Upload folder using huggingface_hub

patches/0001-lazy-slot-context-defer.patch

@@ -0,0 +1,247 @@
+From: opencoti
+Subject: [PATCH 1/N] lazy slot-context allocation — Phase 1 (deferred zero-fill)
+
+Phase 1 of the F4 M3 lazy slot-context rollout. The eager
+`ggml_backend_buffer_clear(buf, 0)` memset that commits every KV
+buffer page to RSS at construction time is split out into a new
+`ensure_cleared()` method invoked from inference entry points
+(init_batch, prepare, update, state_read).
+
+The buffer's malloc happens eagerly at construction time so the
+backend scheduler's `sched_reserve` (which runs at llama_context
+construction, before any batch) can resolve `tensor->buffer` for
+every KV tensor. On Linux, malloc is essentially free for unused
+ranges — it reserves address space without committing pages.
+Pages only commit to RSS when first written. The eager memset(0)
+was what wrote every page, so removing it from the constructor
+moves the RSS commit to the first real inference call.
+
+Effect for the production embedder (qwen3-0.6b,
+`--parallel 2 --ctx-size 16384`, CPU mode): the KV cache's
+~1,792 MiB no longer commits at process startup. Idle RSS drops
+from ~3,478 MiB to roughly model + compute scratch only. First
+request pays the memset cost (~1 s for 1.8 GiB), then warm-state
+RSS matches the un-patched baseline. Phase 1 alone is a
+cold-start win; Phases 2 (grow-on-demand) and 3 (shrink-on-idle)
+target warm-state RSS.
+
+Smoke validation (Qwen2-Math-1.5B Q4_K_M, --parallel 2 -c 4096
+CPU mode): startup emits
+
+    llama_kv_cache:        CPU KV buffer size =   112.00 MiB
+    llama_kv_cache: KV cache zero-fill deferred until first batch
+
+First /v1/chat/completions request emits
+
+    ensure_cleared:        CPU KV buffer zero-fill =  112.00 MiB (deferred)
+
+and produces the expected response.
+
+Coverage of ensure_cleared() call sites:
+- init_batch (inference entry)
+- prepare (iswa wrapper calls prepare() directly on each inner
+  kv_cache without going through init_batch; see
+  llama-kv-cache-iswa.cpp:126-159)
+- update (cross-stream seq_cp resolution + RoPE shift)
+- state_read (saved-state restore writes tensor data)
+
+state_write is left const + un-cleared-safe: empty v_cells write
+zero-cell-count records that state_read's
+`if (cell_count == 0) continue` already handles.
+
+clear(data=true) on an un-cleared cache skips the buffer memset
+deliberately — v_cells.reset() above already represents "no
+data," and committing RSS just to zero it would defeat Phase 1's
+purpose.
+
+The no_alloc model path is unchanged: zero-byte dummy buffers
+are assigned per tensor at construction and trivially "cleared"
+(zero-byte memset). `cleared` is set to true in that case.
+
+Bench: docs/decisions/0001-lazy-slot-context.md §6.2 Phase 1 row.
+Milestone: F4 M3 — see docs/features/memory_embedder.md
+Design: docs/decisions/0001-lazy-slot-context.md
+Upstreaming: deferred; propose to llamafile + llama.cpp
+  maintainers after F4 M3's full phase-set ships and bench
+  numbers are public. This is a generally-useful patch.
+
+---
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -250,7 +250,15 @@ llama_kv_cache::llama_kv_cache(
+         }
+     }
+ 
+-    // allocate tensors and initialize the buffers to avoid NaNs in the padding
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // Eager malloc, deferred zero-fill. The address-space reservation done
++    // by ggml_backend_alloc_ctx_tensors_from_buft is essentially free on
++    // Linux — pages are not committed to RSS until they are first written.
++    // The `ggml_backend_buffer_clear(buf, 0)` memset below is what commits
++    // every page. By skipping it for the non-no_alloc path we keep the
++    // scheduler-visible tensor->buffer wiring intact (sched_reserve runs
++    // at construction time and asserts buffer_id >= 0) while moving the
++    // RSS commit to the first ensure_cleared() call.
+     for (auto & [buft, ctx] : ctx_map) {
+         ggml_backend_buffer_t buf;
+         if (model.hparams.no_alloc) {
+@@ -259,7 +267,7 @@ llama_kv_cache::llama_kv_cache(
+                 t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
+             }
+         } else {
+-            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
++            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet)
+         }
+         if (!buf) {
+             throw std::runtime_error("failed to allocate buffer for kv cache");
+@@ -267,10 +275,23 @@ llama_kv_cache::llama_kv_cache(
+ 
+         LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+ 
+-        ggml_backend_buffer_clear(buf, 0);
++        if (model.hparams.no_alloc) {
++            // Dummy buffer is size 0; clearing is a no-op but keeps the
++            // upstream invariant that constructed buffers are zeroed.
++            ggml_backend_buffer_clear(buf, 0);
++        }
++        // else: leave the buffer un-cleared. ensure_cleared() will zero it
++        // on the first inference entry point that touches tensor data.
++
+         ctxs_bufs.emplace_back(std::move(ctx), buf);
+     }
+ 
++    if (model.hparams.no_alloc) {
++        cleared = true;
++    } else {
++        LLAMA_LOG_INFO("%s: KV cache zero-fill deferred until first batch\n", __func__);
++    }
++
+     {
+         const size_t memory_size_k = size_k_bytes();
+         const size_t memory_size_v = size_v_bytes();
+@@ -327,6 +348,22 @@ llama_kv_cache::llama_kv_cache(
+     debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0;
+ }
+ 
++// opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++void llama_kv_cache::ensure_cleared() {
++    if (cleared) {
++        return;
++    }
++
++    for (auto & [_, buf] : ctxs_bufs) {
++        LLAMA_LOG_INFO("%s: %10s KV buffer zero-fill = %8.2f MiB (deferred)\n", __func__,
++                ggml_backend_buffer_name(buf.get()),
++                ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0);
++        ggml_backend_buffer_clear(buf.get(), 0);
++    }
++
++    cleared = true;
++}
++
+ void llama_kv_cache::clear(bool data) {
+     for (uint32_t s = 0; s < n_stream; ++s) {
+         v_cells[s].reset();
+@@ -334,8 +371,14 @@ void llama_kv_cache::clear(bool data) {
+     }
+ 
+     if (data) {
+-        for (auto & [_, buf] : ctxs_bufs) {
+-            ggml_backend_buffer_clear(buf.get(), 0);
++        // opencoti F4 M3 Phase 1: if the deferred zero-fill has not happened
++        // yet there is nothing to clear — v_cells.reset() above already
++        // satisfies the logical "data cleared" state. Don't commit RSS just
++        // to zero pages that have never been written.
++        if (cleared) {
++            for (auto & [_, buf] : ctxs_bufs) {
++                ggml_backend_buffer_clear(buf.get(), 0);
++            }
+         }
+     }
+ }
+@@ -630,6 +673,9 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
+             bool embd_all) {
+     GGML_UNUSED(embd_all);
+ 
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    ensure_cleared();
++
+     do {
+         balloc.split_reset();
+ 
+@@ -674,6 +720,12 @@ llama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool
+ }
+ 
+ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ubatch> & ubatches) {
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // The iswa wrapper calls prepare() directly on each inner kv_cache without
++    // going through init_batch(), so the ensure_cleared() in init_batch alone
++    // is not enough. Calling here covers both paths and is idempotent.
++    ensure_cleared();
++
+     llama_kv_cache::slot_info_vec_t res;
+ 
+     struct state_t {
+@@ -744,6 +796,15 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co
+ 
+     auto * sched = lctx->get_sched();
+ 
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // update() can be reached without init_batch first (e.g. update-only paths
++    // for cross-stream copy or RoPE shift). When that happens with the buffer
++    // still deferred we have nothing to copy or shift — make the dependency
++    // explicit so the tensor->data dereferences below are safe.
++    if (!sc_info.empty() || do_shift) {
++        ensure_cleared();
++    }
++
+     if (!sc_info.empty()) {
+         assert(n_stream > 1 && "stream copy should never happen with a single stream");
+ 
+@@ -1900,6 +1961,11 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama
+ 
+     GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()));
+ 
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // state_read writes the saved KV blob into our tensors, which requires
++    // backing storage. Force the deferred allocation before we read.
++    ensure_cleared();
++
+     uint32_t n_stream_cur;
+     io.read(&n_stream_cur, sizeof(n_stream_cur));
+     if (n_stream_cur != n_stream) {
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -152,6 +152,16 @@ public:
+ 
+     bool get_has_shift() const;
+ 
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // Idempotent: zeroes the KV-cache backend buffers on first call.
++    // The buffers are malloc'd eagerly at construction (so the scheduler
++    // can resolve tensor->buffer at reserve time) but NOT cleared — on
++    // Linux, malloc reserves address space without committing pages, so
++    // RSS stays low until something writes. This method does the memset
++    // that commits the pages. Called from inference entry points
++    // (init_batch, prepare, update, state_read).
++    void ensure_cleared();
++
+     ggml_type type_k() const;
+     ggml_type type_v() const;
+ 
+@@ -256,6 +266,13 @@ private:
+     // ggml contexts for the KV cache along with the allocated backend buffers:
+     std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
+ 
++    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
++    // True once the backend buffers in ctxs_bufs have been zero-filled.
++    // The constructor sets this true for no_alloc models (whose dummy
++    // buffers are size 0 — clearing is a no-op) and false otherwise. The
++    // first ensure_cleared() call zeroes the buffers and flips this true.
++    bool cleared = false;
++
+     // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
+     // note: this is not part of the KV state and it's only used to speed-up the find_slot() method
+     std::vector<uint32_t> v_heads;

patches/0002-lazy-slot-context-grow.patch

@@ -0,0 +1,596 @@
+From: opencoti
+Subject: [PATCH 2/N] lazy slot-context allocation — Phase 2 (grow on demand + partial zero-fill)
+
+Phase 2 of the F4 M3 lazy slot-context rollout. Builds on Phase 1
+(deferred zero-fill) by adding a soft allocation cap that
+find_slot honors, a grow-on-demand path that bumps the cap when
+real prompts need more cells, and a partial-range zero-fill so
+RSS commits scale with actual workload — not with the configured
+n_ctx_slot × slot_count product.
+
+The end-state for the production embedder
+(qwen3-0.6b, --parallel 2 --ctx-size 16384, CPU mode):
+- With --slot-initial-ctx 0 (the default) Phase 2 is a no-op
+  layered on Phase 1: cap == n_ctx_seq from construction; full
+  buffer cleared on first batch (matches Phase 1 RSS profile).
+- With --slot-initial-ctx 4096 (the recommended embedder
+  setting) the warm RSS scales with whatever cells real prompts
+  have touched. A run that only ever embeds 8–56-token requests
+  commits ~4096 × per-cell bytes, not the full 16384.
+
+The patch threads `slot_initial_ctx` through:
+- public API: llama_context_params (include/llama.h)
+- internal:   llama_cparams (src/llama-cparams.h)
+- ctor chain: llama_context::ctor (src/llama-context.cpp)
+              llama_kv_cache::ctor (src/llama-kv-cache.{h,cpp})
+              llama_kv_cache_iswa::ctor (src/llama-kv-cache-iswa.{h,cpp})
+              llama_model::create_memory call sites (src/llama-model.cpp)
+- hybrid mem: src/llama-memory-hybrid.cpp + -iswa.cpp pass 0
+              (kept on Phase-1 behavior — recurrent/hybrid models
+              are not the F4 embedder target)
+- CLI:        common_params.slot_initial_ctx + --slot-initial-ctx
+              (common/common.{h,cpp}, common/arg.cpp)
+
+Mechanism (inside llama_kv_cache):
+- kv_size_max:    hard cap (== constructor's kv_size).
+- kv_size_alloc:  soft cap; cells [0, kv_size_alloc) visible to
+                  find_slot. Starts at min(slot_initial_ctx,
+                  kv_size_max) — Phase 1 default of 0 collapses
+                  to kv_size_max so find_slot sees everything.
+- n_cells_cleared: half-open range of cells already zero-filled.
+                  Phase 1's `bool cleared` is replaced; the cells
+                  beyond n_cells_cleared have malloc'd address
+                  space but uncommitted pages.
+- ensure_cleared(up_to):
+    Partial memset of cells [n_cells_cleared, up_to) on each
+    layer's K and V tensor across every stream, using
+    ggml_backend_tensor_memset (host-or-device dispatch). The
+    common inference path passes up_to=0 which expands to the
+    current soft cap; state_read passes up_to=kv_size_max for
+    full coverage on the deserialize path.
+- ensure_capacity(cells_needed):
+    Geometric grow (next-pow2, n_pad-aligned) of kv_size_alloc
+    up to kv_size_max. Monotonic — never shrinks (Phase 3 adds
+    shrink-on-idle).
+- prepare(ubatches):
+    When find_slot fails inside the current soft cap, retries
+    after ensure_capacity(used_max + ubatch.n_tokens) +
+    ensure_cleared(). Bounded by kv_size_max; a genuine
+    overflow at the hard cap propagates up unchanged.
+
+Bench: docs/decisions/0001-lazy-slot-context.md §6.2 Phase 2 row.
+Milestone: F4 M3 — see docs/features/memory_embedder.md
+Design: docs/decisions/0001-lazy-slot-context.md
+Upstreaming: deferred; propose to llamafile + llama.cpp
+  maintainers after F4 M3's full phase-set ships and bench
+  numbers are public.
+
+---
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1361,6 +1361,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.kv_unified = value;
+         }
+     ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
++    add_opt(common_arg(
++        // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++        {"--slot-initial-ctx"}, "N",
++        string_format("initial KV-cell allocation per sequence (default: %d, 0 = full n_ctx_seq, "
++                      "positive values cap initial allocation and enable grow-on-demand)",
++                      params.slot_initial_ctx),
++        [](common_params & params, int value) {
++            params.slot_initial_ctx = value;
++        }
++    ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL}));
+     add_opt(common_arg(
+         {"--cache-idle-slots"},
+         {"--no-cache-idle-slots"},
+diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp
+--- a/llama.cpp/common/common.cpp
++++ b/llama.cpp/common/common.cpp
+@@ -1628,6 +1628,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
+ 
+     cparams.n_ctx             = params.n_ctx;
+     cparams.n_seq_max         = params.n_parallel;
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    cparams.slot_initial_ctx  = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx;
+     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+     cparams.n_batch           = params.n_batch;
+     cparams.n_ubatch          = params.n_ubatch;
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -425,6 +425,11 @@ struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
+ struct common_params {
+     int32_t n_predict             =    -1; // max. number of new tokens to predict, -1 == no limit
+     int32_t n_ctx                 =     0; // context size, 0 == context the model was trained with
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // Initial KV-cell allocation per sequence. 0 = full n_ctx_seq allocation
++    // (Phase 1 behavior — only zero-fill is deferred). Positive values cap
++    // the initial allocation and enable grow-on-demand up to n_ctx_seq.
++    int32_t slot_initial_ctx      =     0;
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -338,6 +338,14 @@ extern "C" {
+         uint32_t n_batch;           // logical maximum batch size that can be submitted to llama_decode
+         uint32_t n_ubatch;          // physical maximum batch size
+         uint32_t n_seq_max;         // max number of sequences (i.e. distinct states for recurrent models)
++        // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++        // Initial KV-cell allocation per sequence. 0 = behave like Phase 1
++        // (allocate the full n_ctx_seq up front, only zero-fill is deferred).
++        // Positive values cap the initial allocation at min(slot_initial_ctx,
++        // n_ctx_seq); subsequent prompts that exceed this cap trigger a grow
++        // up to next_pow2 (still capped at n_ctx_seq). Reduces warm-state RSS
++        // when typical prompts are much shorter than n_ctx_seq.
++        uint32_t slot_initial_ctx;
+         uint32_t n_rs_seq;          // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
+         int32_t  n_threads;         // number of threads to use for generation
+         int32_t  n_threads_batch;   // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -58,6 +58,9 @@ llama_context::llama_context(
+         cparams.n_rs_seq = 0;
+     }
+ 
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    cparams.slot_initial_ctx = params.slot_initial_ctx;
++
+     cparams.n_threads        = params.n_threads;
+     cparams.n_threads_batch  = params.n_threads_batch;
+     cparams.yarn_ext_factor  = params.yarn_ext_factor  >= 0.0f ? params.yarn_ext_factor  : hparams.yarn_ext_factor;
+@@ -3346,6 +3349,7 @@ llama_context_params llama_context_default_params() {
+         /*.n_batch                     =*/ 2048,
+         /*.n_ubatch                    =*/ 512,
+         /*.n_seq_max                   =*/ 1,
++        /*.slot_initial_ctx            =*/ 0,
+         /*.n_rs_seq                    =*/ 0,
+         /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+         /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -12,6 +12,10 @@ struct llama_cparams {
+     uint32_t n_batch;
+     uint32_t n_ubatch;
+     uint32_t n_seq_max;
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // 0 = full n_ctx_seq allocation. Positive value caps the initial KV-cell
++    // allocation per sequence and enables grow-on-demand up to n_ctx_seq.
++    uint32_t slot_initial_ctx;
+     uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+     int32_t  n_threads;       // number of threads to use for generation
+     int32_t  n_threads_batch; // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp
+--- a/llama.cpp/src/llama-kv-cache-iswa.cpp
++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp
+@@ -20,6 +20,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+                      bool   swa_full,
+                      bool   unified,
+                  uint32_t   kv_size,
++                 uint32_t   kv_size_initial,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_ubatch,
+                  uint32_t   n_pad,
+@@ -61,14 +62,14 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+ 
+     kv_base = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+-            v_trans, offload, unified, size_base, n_seq_max, n_pad,
++            v_trans, offload, unified, size_base, kv_size_initial, n_seq_max, n_pad,
+             0, LLAMA_SWA_TYPE_NONE, filter_base, reuse);
+ 
+     LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
+ 
+     kv_swa = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+-            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
++            v_trans, offload, unified, size_swa, kv_size_initial, n_seq_max, n_pad,
+             hparams.n_swa, hparams.swa_type, filter_swa, reuse);
+ }
+ 
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h
+--- a/llama.cpp/src/llama-kv-cache-iswa.h
++++ b/llama.cpp/src/llama-kv-cache-iswa.h
+@@ -22,6 +22,9 @@ public:
+                          bool   swa_full,
+                          bool   unified,
+                      uint32_t   kv_size,
++                     // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++                     // Forwarded to both kv_base and kv_swa. 0 = full kv_size.
++                     uint32_t   kv_size_initial,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_ubatch,
+                      uint32_t   n_pad,
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -85,6 +85,7 @@ llama_kv_cache::llama_kv_cache(
+                      bool   offload,
+                      bool   unified,
+                  uint32_t   kv_size,
++                 uint32_t   kv_size_initial,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_pad,
+                  uint32_t   n_swa,
+@@ -96,6 +97,24 @@ llama_kv_cache::llama_kv_cache(
+ 
+     GGML_ASSERT(kv_size % n_pad == 0);
+ 
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // Initialize the hard cap and the soft cap. kv_size_initial==0 (the
++    // legacy / Phase-1 default) means "no soft cap" so the whole kv_size
++    // is visible to find_slot. A positive value is clamped to [n_pad,
++    // kv_size] and rounded up to the next n_pad multiple.
++    kv_size_max = kv_size;
++    if (kv_size_initial == 0) {
++        kv_size_alloc = kv_size_max;
++    } else {
++        uint32_t cap = std::min(kv_size_initial, kv_size_max);
++        cap = std::max(cap, n_pad);
++        cap = ((cap + n_pad - 1) / n_pad) * n_pad;
++        cap = std::min(cap, kv_size_max);
++        kv_size_alloc = cap;
++        LLAMA_LOG_INFO("%s: slot_initial_ctx = %u cells (cap = %u)\n",
++                __func__, kv_size_alloc, kv_size_max);
++    }
++
+     const uint32_t n_layer_kv = hparams.n_layer_kv();
+ 
+     // define a comparator for the buft -> ctx map to ensure that the order is well-defined:
+@@ -287,7 +306,9 @@ llama_kv_cache::llama_kv_cache(
+     }
+ 
+     if (model.hparams.no_alloc) {
+-        cleared = true;
++        // opencoti F4 M3 Phase 1+2 — dummy buffers are size 0; treat all
++        // cells as "already cleared" so ensure_cleared is a no-op.
++        n_cells_cleared = kv_size_max;
+     } else {
+         LLAMA_LOG_INFO("%s: KV cache zero-fill deferred until first batch\n", __func__);
+     }
+@@ -348,20 +369,81 @@ llama_kv_cache::llama_kv_cache(
+     debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0;
+ }
+ 
+-// opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+-void llama_kv_cache::ensure_cleared() {
+-    if (cleared) {
++// opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md
++void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) {
++    // up_to_cells == 0 means "clear up to the current soft cap". Callers
++    // that need full coverage (state_read) pass kv_size_max.
++    const uint32_t target = up_to_cells == 0
++        ? kv_size_alloc
++        : std::min(up_to_cells, kv_size_max);
++
++    if (n_cells_cleared >= target) {
+         return;
+     }
+ 
+-    for (auto & [_, buf] : ctxs_bufs) {
+-        LLAMA_LOG_INFO("%s: %10s KV buffer zero-fill = %8.2f MiB (deferred)\n", __func__,
+-                ggml_backend_buffer_name(buf.get()),
+-                ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0);
+-        ggml_backend_buffer_clear(buf.get(), 0);
++    const uint32_t a = n_cells_cleared;
++    const uint32_t b = target;
++
++    // Per-layer, per-stream partial memset of the K and V tensors.
++    // ggml_backend_tensor_memset handles the host/device dispatch — for the
++    // CPU backend (the only one the embedder uses today) this is a direct
++    // memset on the malloc'd buffer; for offloaded buffers it's a backend
++    // op. Either way only bytes for cells [a, b) get touched, so on Linux
++    // only those pages get committed to RSS.
++    for (const auto & layer : layers) {
++        for (uint32_t st = 0; st < n_stream; ++st) {
++            if (layer.k) {
++                const size_t off = (size_t) st * (size_t) layer.k->nb[2]
++                                 + (size_t) a  * (size_t) layer.k->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) layer.k->nb[1];
++                ggml_backend_tensor_memset(layer.k, 0, off, sz);
++            }
++            if (layer.v) {
++                const size_t off = (size_t) st * (size_t) layer.v->nb[2]
++                                 + (size_t) a  * (size_t) layer.v->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) layer.v->nb[1];
++                ggml_backend_tensor_memset(layer.v, 0, off, sz);
++            }
++        }
++    }
++
++    LLAMA_LOG_INFO("%s: cleared KV cells [%u, %u) of %u (%s)\n",
++            __func__, a, b, kv_size_max,
++            up_to_cells == 0 ? "soft cap" : "full");
++
++    n_cells_cleared = b;
++}
++
++// opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) {
++    if (cells_needed <= kv_size_alloc) {
++        return true;
++    }
++
++    if (kv_size_alloc >= kv_size_max) {
++        return false;
++    }
++
++    // Grow geometrically (next power-of-two) and align up to n_pad. The
++    // pow2 ramp avoids re-growing every few tokens for long prompts; n_pad
++    // alignment preserves the constructor's GGML_ASSERT(kv_size % n_pad == 0)
++    // invariant for find_slot.
++    uint32_t new_cap = std::max<uint32_t>(kv_size_alloc, n_pad);
++    while (new_cap < cells_needed && new_cap < kv_size_max) {
++        new_cap *= 2;
+     }
++    new_cap = ((new_cap + n_pad - 1) / n_pad) * n_pad;
++    new_cap = std::min(new_cap, kv_size_max);
+ 
+-    cleared = true;
++    if (new_cap == kv_size_alloc) {
++        return cells_needed <= kv_size_alloc;
++    }
++
++    LLAMA_LOG_INFO("%s: grow soft cap %u -> %u (need %u, max %u)\n",
++            __func__, kv_size_alloc, new_cap, cells_needed, kv_size_max);
++
++    kv_size_alloc = new_cap;
++    return cells_needed <= kv_size_alloc;
+ }
+ 
+ void llama_kv_cache::clear(bool data) {
+@@ -371,14 +453,16 @@ void llama_kv_cache::clear(bool data) {
+     }
+ 
+     if (data) {
+-        // opencoti F4 M3 Phase 1: if the deferred zero-fill has not happened
+-        // yet there is nothing to clear — v_cells.reset() above already
+-        // satisfies the logical "data cleared" state. Don't commit RSS just
+-        // to zero pages that have never been written.
+-        if (cleared) {
+-            for (auto & [_, buf] : ctxs_bufs) {
+-                ggml_backend_buffer_clear(buf.get(), 0);
+-            }
++        // opencoti F4 M3 Phase 1+2: if no zero-fill has happened yet there
++        // is nothing to clear — v_cells.reset() above already satisfies the
++        // logical "data cleared" state. Don't commit RSS just to zero pages
++        // that have never been written. When a partial clear is already on
++        // the books, re-issue it against the current soft cap so any cells
++        // touched between the first ensure_cleared and now are re-zeroed.
++        if (n_cells_cleared > 0) {
++            const uint32_t b = n_cells_cleared;
++            n_cells_cleared = 0;
++            ensure_cleared(b);
+         }
+     }
+ }
+@@ -743,10 +827,28 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_
+ 
+     for (const auto & ubatch : ubatches) {
+         // only find a suitable slot for the ubatch. don't modify the cells yet
+-        const auto sinfo_new = find_slot(ubatch, false);
++        auto sinfo_new = find_slot(ubatch, false);
+         if (sinfo_new.empty()) {
+-            success = false;
+-            break;
++            // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++            // Phase 2 keeps the soft alloc cap (kv_size_alloc) below kv_size_max
++            // until a real ubatch needs more cells than the cap exposes. When
++            // find_slot fails, try to grow the cap to cover what we already
++            // know is used plus the ubatch's token count, then retry. The grow
++            // is bounded by kv_size_max; if we are already at the hard cap the
++            // failure is genuine and we propagate it up.
++            uint32_t used_max = 0;
++            for (uint32_t st = 0; st < n_stream; ++st) {
++                used_max = std::max(used_max, v_cells[st].used_max_p1());
++            }
++            const uint32_t want = used_max + ubatch.n_tokens;
++            if (ensure_capacity(want)) {
++                ensure_cleared();
++                sinfo_new = find_slot(ubatch, false);
++            }
++            if (sinfo_new.empty()) {
++                success = false;
++                break;
++            }
+         }
+ 
+         // remember the position that we found
+@@ -981,14 +1083,20 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
+ 
+         uint32_t head_cur = v_heads[seq_to_stream[seq_id]];
+ 
++        // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++        // The Phase-1 search swept the full hard cap (cells.size() == kv_size_max).
++        // Phase 2 caps it at the current soft alloc; prepare() grows the cap and
++        // retries when the soft cap is too small for the requested ubatch.
++        const uint32_t cap = kv_size_alloc;
++
+         // if we have enough unused cells before the current head ->
+         //   better to start searching from the beginning of the cache, hoping to fill it
+         if (head_cur > cells.get_used() + 2*n_tokens) {
+             head_cur = 0;
+         }
+ 
+-        if (n_tokens > cells.size()) {
+-            LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size());
++        if (n_tokens > cap) {
++            // not necessarily fatal — prepare() may grow the cap and retry
+             return { };
+         }
+ 
+@@ -999,8 +1107,8 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
+         const uint32_t n_test = cont ? n_tokens : 1;
+ 
+         while (true) {
+-            if (head_cur + n_test > cells.size()) {
+-                n_tested += cells.size() - head_cur;
++            if (head_cur + n_test > cap) {
++                n_tested += cap - head_cur;
+                 head_cur = 0;
+                 continue;
+             }
+@@ -1058,7 +1166,7 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
+                 res.idxs[s].clear();
+             }
+ 
+-            if (n_tested >= cells.size()) {
++            if (n_tested >= cap) {
+                 //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens);
+                 return { };
+             }
+@@ -1961,10 +2069,13 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama
+ 
+     GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()));
+ 
+-    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+-    // state_read writes the saved KV blob into our tensors, which requires
+-    // backing storage. Force the deferred allocation before we read.
+-    ensure_cleared();
++    // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md
++    // state_read writes the saved KV blob into our tensors at arbitrary
++    // offsets, so we cannot rely on the soft cap. Force a full-range
++    // zero-fill before we read — this commits all KV pages but only on
++    // the explicit deserialize path.
++    ensure_capacity(kv_size_max);
++    ensure_cleared(kv_size_max);
+ 
+     uint32_t n_stream_cur;
+     io.read(&n_stream_cur, sizeof(n_stream_cur));
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -101,6 +101,11 @@ public:
+                          bool   offload,
+                          bool   unified,
+                      uint32_t   kv_size,
++                     // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++                     // Initial allocation cap. 0 means the same as kv_size
++                     // (Phase 1 behavior). Otherwise clamped to [n_pad, kv_size]
++                     // and rounded up to the next n_pad multiple.
++                     uint32_t   kv_size_initial,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_pad,
+                      uint32_t   n_swa,
+@@ -152,15 +157,28 @@ public:
+ 
+     bool get_has_shift() const;
+ 
+-    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+-    // Idempotent: zeroes the KV-cache backend buffers on first call.
+-    // The buffers are malloc'd eagerly at construction (so the scheduler
+-    // can resolve tensor->buffer at reserve time) but NOT cleared — on
+-    // Linux, malloc reserves address space without committing pages, so
+-    // RSS stays low until something writes. This method does the memset
+-    // that commits the pages. Called from inference entry points
+-    // (init_batch, prepare, update, state_read).
+-    void ensure_cleared();
++    // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md
++    // Idempotent. Zeroes cells [n_cells_cleared, up_to_cells) of every KV
++    // layer's K and V tensors across every stream. The buffers are malloc'd
++    // eagerly at construction (so the scheduler can resolve tensor->buffer
++    // at reserve time) but their pages are not committed to RSS until they
++    // are first written. Partial memset (via ggml_backend_tensor_memset)
++    // commits only the requested range — that is the Phase 2 RSS win.
++    //
++    // up_to_cells == 0 means "use the current soft cap kv_size_alloc"
++    // (the normal inference path). Pass kv_size_max for paths that need
++    // full coverage (state_read).
++    void ensure_cleared(uint32_t up_to_cells = 0);
++
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // Grow the soft allocation cap. Returns true iff the cap covers
++    // cells_needed after the grow. Bound by kv_size_max. Idempotent at
++    // the cap.
++    bool ensure_capacity(uint32_t cells_needed);
++
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // The current soft cap (cells visible to find_slot).
++    uint32_t get_kv_size_alloc() const { return kv_size_alloc; }
+ 
+     ggml_type type_k() const;
+     ggml_type type_v() const;
+@@ -266,12 +284,22 @@ private:
+     // ggml contexts for the KV cache along with the allocated backend buffers:
+     std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
+ 
+-    // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+-    // True once the backend buffers in ctxs_bufs have been zero-filled.
+-    // The constructor sets this true for no_alloc models (whose dummy
+-    // buffers are size 0 — clearing is a no-op) and false otherwise. The
+-    // first ensure_cleared() call zeroes the buffers and flips this true.
+-    bool cleared = false;
++    // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md
++    // Cells in the half-open range [0, n_cells_cleared) are guaranteed to
++    // be zeroed across every layer's K and V tensors and every stream.
++    // ensure_cleared(up_to) grows this monotonically up to kv_size_alloc
++    // (or kv_size_max for the explicit-full call). The constructor sets
++    // this to kv_size_max for no_alloc models (whose dummy buffers are
++    // size 0 — clearing is a no-op) and 0 otherwise.
++    uint32_t n_cells_cleared = 0;
++
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // kv_size_max:   hard cap on cells (set from constructor's kv_size).
++    // kv_size_alloc: current soft cap. find_slot only scans cells [0, kv_size_alloc).
++    //                Grows via ensure_capacity, never shrinks (Phase 2; Phase 3
++    //                adds shrink-on-idle).
++    uint32_t kv_size_max   = 0;
++    uint32_t kv_size_alloc = 0;
+ 
+     // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
+     // note: this is not part of the KV state and it's only used to speed-up the find_slot() method
+diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp
++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+@@ -40,6 +40,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
+         swa_full,
+         unified,
+         kv_size,
++        // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++        // Hybrid-iswa does not yet thread slot_initial_ctx; pass 0 for
++        // Phase 1 backward-compat (full eager allocation).
++        0,
+         n_seq_max,
+         n_ubatch,
+         n_pad,
+diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp
+--- a/llama.cpp/src/llama-memory-hybrid.cpp
++++ b/llama.cpp/src/llama-memory-hybrid.cpp
+@@ -39,6 +39,10 @@ llama_memory_hybrid::llama_memory_hybrid(
+         offload,
+         unified,
+         kv_size,
++        // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++        // Hybrid memory does not yet thread slot_initial_ctx; pass 0 for
++        // Phase 1 backward-compat (full eager allocation).
++        0,
+         n_seq_max,
+         n_pad,
+         n_swa,
+diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp
+--- a/llama.cpp/src/llama-model.cpp
++++ b/llama.cpp/src/llama-model.cpp
+@@ -2073,6 +2073,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 params.swa_full,
+                                 cparams.kv_unified,
+                                 cparams.n_ctx_seq,
++                                // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++                                cparams.slot_initial_ctx,
+                                 cparams.n_seq_max,
+                                 cparams.n_ubatch,
+                                 1,
+@@ -2089,6 +2091,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 cparams.offload_kqv,
+                                 cparams.kv_unified,
+                                 cparams.n_ctx_seq,
++                                // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++                                cparams.slot_initial_ctx,
+                                 cparams.n_seq_max,
+                                 1,
+                                 hparams.n_swa,

patches/0003-arg-assertion-bypass.patch

@@ -0,0 +1,46 @@
+From: opencoti
+Subject: [PATCH 3/N] bypass upstream `params.n_gpu_layers < 0` assertions
+
+Upstream llama.cpp pin 5e9c63546 has TWO `GGML_ASSERT(n_gpu_layers < 0)`
+calls in common/arg.cpp:
+
+1. At the `-ngl` option registration site (params.n_gpu_layers).
+2. At the `-ngld` (draft) option registration site
+   (params.speculative.n_gpu_layers).
+
+Both fire on every `common_params_parser_init` call regardless of args,
+making the patched binary unable to start in `--server`, `--cli`, or
+`--chat` mode. Mozilla-Ocho v1.4.0 (pre-assertion) was unaffected.
+
+Each assertion is a "display-format guard" for the next line, which
+maps `n_gpu_layers == -1` to "auto" and anything else to "all" in the
+help text. The fix is to either extend the format to handle non-
+negative defaults, or to drop the assertion. We drop it — the
+display "all" for a positive default is a minor cosmetic issue,
+not a correctness problem.
+
+This is an upstream-bug-bypass patch, not opencoti scope. To be
+proposed upstream once we surface a minimal reproducer.
+
+Bench: closes the F4 M3 bench path that requires booting the
+patched binary in --server mode.
+Milestone: F4 M3 — see docs/features/memory_embedder.md
+Upstreaming: TBD — propose to llama.cpp once a minimal repro is
+  in hand. Likely already fixed in newer upstream commits.
+
+---
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -2362,7 +2362,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             }
+         }
+     ).set_env("LLAMA_ARG_N_CPU_MOE"));
+-    GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0
++    // opencoti F4 M3 patch 0003 — see docs/decisions/0001-lazy-slot-context.md
++    // Upstream assertion fires on every common_params_parser_init invocation
++    // (display-format guard for the next line). Dropped — display says "all"
++    // for non-negative defaults, which is cosmetic, not a correctness bug.
+     add_opt(common_arg(
+         {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
+         string_format("max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: %s)", params.n_gpu_layers == -1 ? "auto" : "all"),

patches/0004-cuda-std-cpp17.patch

@@ -0,0 +1,50 @@
+From: opencoti
+Subject: [PATCH 4/N] add -std=c++17 to ggml-cuda NVCC flags
+
+Mozilla-Ocho's `llamafile/cuda.sh` builds the `ggml-cuda.so` DSO that
+the llamafile runtime loads when the user passes `-ngl <N>`. The
+ggml-cuda sources have moved to C++17 features (`std::is_same_v`, fold
+expressions, structured bindings), but cuda.sh's COMMON_FLAGS do not
+pass `-std=c++17` to nvcc.
+
+For CUDA 11.x and older 12.x toolkits this worked because nvcc
+defaulted to C++14 and the sources were C++14-clean. Newer CUDA
+toolkits ship CCCL that requires C++17 explicitly via a #error
+guard, so building against CUDA 12.6+ (and definitely CUDA 13.x)
+fails with `error: namespace "std" has no member "is_same_v"` and
+~30 follow-on errors per .cu translation unit.
+
+This patch adds `-std=c++17` to COMMON_FLAGS so cuda.sh builds
+against any reasonable CUDA toolkit. The host compiler is already
+on C++23 via cosmocc, so the host-flag side is fine; this is
+purely the nvcc device-flag side.
+
+Bench: closes the GPU validation path. Without this, the patched
+llamafile cannot load ggml-cuda.so on solidPC (and on any host
+where CUDA 12.6+ is the local toolkit), and the binary silently
+falls back to CPU even with `-ngl all`.
+Milestone: F4 M3 GPU validation (task #113) — see
+  docs/decisions/0001-lazy-slot-context.md and
+  docs/features/memory_embedder.md
+Upstreaming: TBD — propose to Mozilla-Ocho once they decide on a
+  minimum-supported CUDA toolkit. Currently their build matrix is
+  unclear about whether CUDA 13.x is expected to work; this patch
+  is the minimal fix that does not regress older toolkits.
+
+---
+diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
+--- a/llamafile/cuda.sh
++++ b/llamafile/cuda.sh
+@@ -232,7 +232,12 @@ if [ "$COMPRESS" = "1" ]; then
+ fi
+ 
+ # NVCC compiler flags
++# opencoti note: -std=c++17 added because the ggml-cuda .cu sources use
++# C++17 features (std::is_same_v, structured bindings, fold expressions).
++# Newer CUDA toolkits (12.x+) need this explicitly; older toolkits did C++14
++# by default. Both work with c++17.
+ COMMON_FLAGS="\
++  -std=c++17 \
+   --use_fast_math \
+   --extended-lambda \
+   $EXTRA_INCLUDES \

patches/0005-lazy-slot-context-shrink.patch

@@ -0,0 +1,582 @@
+From: opencoti
+Subject: [PATCH 5/N] lazy slot-context allocation — Phase 3 (shrink on idle)
+
+Phase 2 added a soft cap on per-stream KV cells (kv_size_alloc <= kv_size_max)
+and grow-on-demand via ensure_capacity. The soft cap never shrinks, so once a
+single long prompt has caused it to grow, the embedder's RSS stays at the
+grown size for the lifetime of the process. For an embedder running
+`--parallel 2 --ctx-size 16384` with typical prompts well under 4K cells,
+this means the headline Phase-2 win evaporates after the first long-context
+request.
+
+Phase 3 plugs that hole. When an idle threshold has elapsed since the last
+apply_ubatch:
+
+  1. Cells in the range [kv_size_shrink_target, kv_size_alloc) are evicted
+     across every stream via cells.rm(i). The server-context layer's
+     prompt cache keeps cells alive between embed requests for reuse;
+     decommitting their backing pages without first evicting would leave
+     the prompt cache pointing at zeroed memory and corrupt subsequent
+     re-uses. The eviction makes the shrink unconditional.
+
+  2. The pages backing cells [target, current_alloc) of every layer's K
+     and V tensors are returned to the kernel via
+     madvise(addr, len, MADV_DONTNEED).
+
+  3. n_cells_cleared is rolled back to the shrink target so the next
+     ensure_cleared() partial-memset re-zeros only what comes back.
+
+  4. kv_size_alloc is reset to the shrink target so the next request can
+     grow again from the initial cap.
+
+Why madvise(MADV_DONTNEED) and not posix_madvise(POSIX_MADV_DONTNEED):
+glibc on Linux maps posix_madvise(POSIX_MADV_DONTNEED) to a no-op (see
+glibc sysdeps/posix/posix_madvise.c — the standard explicitly forbids
+"actively destroying" pages, which is what we need). Only the Linux-
+specific madvise(MADV_DONTNEED) actually decommits anonymous pages and
+returns RSS to the kernel. We declared MADV_DONTNEED with the canonical
+Linux value (4) under #ifndef and extern-declared madvise() so cosmocc's
+default feature set (which does not define _GNU_SOURCE and so does not
+expose madvise() through sys/mman.h) builds cleanly.
+
+The shrink check runs at the top of init_batch() and prepare() — both
+entry points to actual KV work. It is gated by:
+  - slot_shrink_idle_ms_ > 0 (feature enabled)
+  - kv_size_alloc > kv_size_shrink_target (something to shrink)
+  - last_active_ms_ != 0 AND (now - last_active_ms_) >= idle threshold
+
+Surface (CLI):
+  --slot-shrink-idle-ms N (default 0 = disabled; positive enables)
+  env LLAMA_ARG_SLOT_SHRINK_IDLE_MS
+
+Surface (C API):
+  llama_context_params::slot_shrink_idle_ms (uint32_t, default 0)
+
+Plumbed through llama_cparams -> llama_kv_cache ctor ->
+(llama_kv_cache_iswa ctor for SWA models) -> llama-model.cpp call sites.
+Backward-compat: llama-memory-hybrid{,-iswa}.cpp pass 0 (no shrink), the
+same way they already pass 0 for kv_size_initial.
+
+Bench (CPU, --gpu disable so KV lives in host pages): the embedder
+configured `--parallel 2 --ctx-size 16384 --pooling last
+--slot-initial-ctx 256 --slot-shrink-idle-ms 30000` starts at 719 MB
+RSS, grows to 1720 MB after a long-prompt corpus pass (soft cap 256 ->
+2048), and after the 30 s idle drops to 1329 MB on the next short embed.
+391.7 MB returned to the kernel — measured by /proc/<pid>/status VmRSS.
+Quality (cosine min vs M2 baseline) stays at 0.999797 across the 48-prompt
+corpus.
+
+Milestone: F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+Upstreaming: TBD — kv_cache idle-shrink is a generally-useful concept; the
+  cell-eviction + madvise(MADV_DONTNEED) approach is not opencoti-specific.
+  Proposal candidate after F4 closes.
+
+---
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1371,6 +1371,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.slot_initial_ctx = value;
+         }
+     ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL}));
++    add_opt(common_arg(
++        // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++        {"--slot-shrink-idle-ms"}, "N",
++        string_format("idle threshold (ms) after which a grown KV soft cap is shrunk back to "
++                      "--slot-initial-ctx and the over-cap pages are decommitted via posix_madvise "
++                      "(default: %d, 0 disables; requires --slot-initial-ctx > 0)",
++                      params.slot_shrink_idle_ms),
++        [](common_params & params, int value) {
++            params.slot_shrink_idle_ms = value;
++        }
++    ).set_env("LLAMA_ARG_SLOT_SHRINK_IDLE_MS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL}));
+     add_opt(common_arg(
+         {"--cache-idle-slots"},
+         {"--no-cache-idle-slots"},
+diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp
+--- a/llama.cpp/common/common.cpp
++++ b/llama.cpp/common/common.cpp
+@@ -1630,6 +1630,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
+     cparams.n_seq_max         = params.n_parallel;
+     // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+     cparams.slot_initial_ctx  = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx;
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms;
+     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+     cparams.n_batch           = params.n_batch;
+     cparams.n_ubatch          = params.n_ubatch;
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -430,6 +430,11 @@ struct common_params {
+     // (Phase 1 behavior — only zero-fill is deferred). Positive values cap
+     // the initial allocation and enable grow-on-demand up to n_ctx_seq.
+     int32_t slot_initial_ctx      =     0;
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Idle threshold (ms) after which a grown soft cap is shrunk back to
++    // slot_initial_ctx and the over-cap pages are decommitted via
++    // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior).
++    int32_t slot_shrink_idle_ms   =     0;
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -346,6 +346,13 @@ extern "C" {
+         // up to next_pow2 (still capped at n_ctx_seq). Reduces warm-state RSS
+         // when typical prompts are much shorter than n_ctx_seq.
+         uint32_t slot_initial_ctx;
++        // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++        // Idle threshold after which a grown soft cap is shrunk back to
++        // slot_initial_ctx and the pages beyond it are returned to the OS via
++        // posix_madvise(POSIX_MADV_DONTNEED). 0 disables shrink-on-idle
++        // (Phase 2 behavior — soft cap grows monotonically). Requires
++        // slot_initial_ctx > 0 to have any visible effect.
++        uint32_t slot_shrink_idle_ms;
+         uint32_t n_rs_seq;          // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
+         int32_t  n_threads;         // number of threads to use for generation
+         int32_t  n_threads_batch;   // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -60,6 +60,8 @@ llama_context::llama_context(
+ 
+     // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+     cparams.slot_initial_ctx = params.slot_initial_ctx;
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms;
+ 
+     cparams.n_threads        = params.n_threads;
+     cparams.n_threads_batch  = params.n_threads_batch;
+@@ -3350,6 +3352,7 @@ llama_context_params llama_context_default_params() {
+         /*.n_ubatch                    =*/ 512,
+         /*.n_seq_max                   =*/ 1,
+         /*.slot_initial_ctx            =*/ 0,
++        /*.slot_shrink_idle_ms         =*/ 0,
+         /*.n_rs_seq                    =*/ 0,
+         /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+         /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -16,6 +16,11 @@ struct llama_cparams {
+     // 0 = full n_ctx_seq allocation. Positive value caps the initial KV-cell
+     // allocation per sequence and enables grow-on-demand up to n_ctx_seq.
+     uint32_t slot_initial_ctx;
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // 0 disables shrink-on-idle. Positive value is the idle threshold in
++    // milliseconds: when a grown KV-cache is idle for at least this long, the
++    // soft cap is reset to slot_initial_ctx and the pages beyond are decommitted.
++    uint32_t slot_shrink_idle_ms;
+     uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+     int32_t  n_threads;       // number of threads to use for generation
+     int32_t  n_threads_batch; // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp
+--- a/llama.cpp/src/llama-kv-cache-iswa.cpp
++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp
+@@ -21,6 +21,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+                      bool   unified,
+                  uint32_t   kv_size,
+                  uint32_t   kv_size_initial,
++                 uint32_t   slot_shrink_idle_ms,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_ubatch,
+                  uint32_t   n_pad,
+@@ -62,15 +63,15 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+ 
+     kv_base = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+-            v_trans, offload, unified, size_base, kv_size_initial, n_seq_max, n_pad,
+-            0, LLAMA_SWA_TYPE_NONE, filter_base, reuse);
++            v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms,
++            n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse);
+ 
+     LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
+ 
+     kv_swa = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+-            v_trans, offload, unified, size_swa, kv_size_initial, n_seq_max, n_pad,
+-            hparams.n_swa, hparams.swa_type, filter_swa, reuse);
++            v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms,
++            n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse);
+ }
+ 
+ void llama_kv_cache_iswa::clear(bool data) {
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h
+--- a/llama.cpp/src/llama-kv-cache-iswa.h
++++ b/llama.cpp/src/llama-kv-cache-iswa.h
+@@ -25,6 +25,9 @@ public:
+                      // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+                      // Forwarded to both kv_base and kv_swa. 0 = full kv_size.
+                      uint32_t   kv_size_initial,
++                     // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++                     // Forwarded to both kv_base and kv_swa. 0 = shrink disabled.
++                     uint32_t   slot_shrink_idle_ms,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_ubatch,
+                      uint32_t   n_pad,
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -7,12 +7,35 @@
+ 
+ #include <algorithm>
+ #include <cassert>
++#include <chrono>
+ #include <cmath>
+ #include <cstring>
+ #include <limits>
+ #include <map>
+ #include <stdexcept>
+ 
++// opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++// On Linux, madvise(addr, len, MADV_DONTNEED) is the call that actually
++// decommits pages: the kernel zeroes the page table entries, releases the
++// physical pages, and the next access faults in fresh zero-filled pages.
++// glibc's posix_madvise(POSIX_MADV_DONTNEED) is documented as advisory and
++// is NOT guaranteed to reduce RSS — empirically it is a no-op on the
++// glibc/Debian-11 setup the embedder runs on. We must call madvise()
++// directly.
++//
++// cosmocc's default headers (without _GNU_SOURCE) do not declare madvise()
++// or MADV_DONTNEED, but the syscall exists on every Linux kernel the
++// embedder runs on. Hardcode the Linux ABI value (asm-generic/mman-common.h:
++// `#define MADV_DONTNEED 4`) and forward-declare madvise() to keep
++// _GNU_SOURCE out of the global include. <unistd.h> provides
++// sysconf(_SC_PAGESIZE) for page-boundary math.
++#include <sys/mman.h>
++#include <unistd.h>
++#ifndef MADV_DONTNEED
++#  define MADV_DONTNEED 4
++#endif
++extern "C" int madvise(void *addr, size_t length, int advice);
++
+ static bool ggml_is_power_of_2(int n) {
+     return (n & (n - 1)) == 0;
+ }
+@@ -86,6 +109,7 @@ llama_kv_cache::llama_kv_cache(
+                      bool   unified,
+                  uint32_t   kv_size,
+                  uint32_t   kv_size_initial,
++                 uint32_t   slot_shrink_idle_ms,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_pad,
+                  uint32_t   n_swa,
+@@ -115,6 +139,18 @@ llama_kv_cache::llama_kv_cache(
+                 __func__, kv_size_alloc, kv_size_max);
+     }
+ 
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // The shrink target is the initial cap we just computed. last_active_ms_
++    // starts at 0 (epoch) so a long pause before the first request still
++    // shrinks (correct behavior — nothing to lose).
++    kv_size_shrink_target = kv_size_alloc;
++    slot_shrink_idle_ms_  = slot_shrink_idle_ms;
++    last_active_ms_       = 0;
++    if (slot_shrink_idle_ms_ > 0 && kv_size_initial > 0 && kv_size_initial < kv_size_max) {
++        LLAMA_LOG_INFO("%s: slot_shrink_idle_ms = %u (target cap = %u)\n",
++                __func__, slot_shrink_idle_ms_, kv_size_shrink_target);
++    }
++
+     const uint32_t n_layer_kv = hparams.n_layer_kv();
+ 
+     // define a comparator for the buft -> ctx map to ensure that the order is well-defined:
+@@ -446,6 +482,151 @@ bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) {
+     return cells_needed <= kv_size_alloc;
+ }
+ 
++// opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++// Decommit pages of every layer's K/V tensors in the byte-range that
++// corresponds to cells [from_cells, to_cells). Returns those pages to the OS
++// — RSS drops by the size of the range, and the next access to those pages
++// will fault them back in as fresh zero-filled pages.
++//
++// Only safe to call after kv_size_alloc has been reset to from_cells (or
++// lower) and n_cells_cleared has been reset accordingly, so the bookkeeping
++// stays in sync with what's actually backed by physical memory.
++//
++// The decommit is per-stream because the KV tensors are 3-D
++// [n_embd, kv_size, n_stream]; cells [a, b) within stream s occupy a
++// contiguous byte-range whose start address is (base + s*nb[2] + a*nb[1])
++// and length is (b-a)*nb[1]. posix_madvise is page-grained — bytes that
++// don't cover a full page are silently kept resident, which is fine for
++// the cells that straddle page boundaries.
++static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream,
++                                          uint32_t from_cells, uint32_t to_cells) {
++    if (!t || to_cells <= from_cells) {
++        return;
++    }
++    ggml_backend_buffer_t buf = t->buffer;
++    if (!buf) {
++        return;
++    }
++    // Only the host (CPU) backend's buffer is malloc'd memory we can
++    // decommit via madvise. For device buffers (CUDA/Metal/Vulkan) the K/V
++    // cache lives in device memory; skip — there are no host-side pages
++    // to return. This means Phase 3's RSS-shrink win applies to host-mode
++    // (e.g. embedder with -ngl 0); GPU-mode binaries still benefit from
++    // the soft-cap reset but RSS doesn't move.
++    if (!ggml_backend_buffer_is_host(buf)) {
++        return;
++    }
++    void * base = ggml_backend_buffer_get_base(buf);
++    if (!base) {
++        return;
++    }
++    const size_t stride_stream = (size_t) t->nb[2];
++    const size_t stride_cell   = (size_t) t->nb[1];
++    const size_t off_in_buf    = (size_t) ((char *) t->data - (char *) base);
++    for (uint32_t s = 0; s < n_stream; ++s) {
++        const size_t off = off_in_buf
++                         + (size_t) s          * stride_stream
++                         + (size_t) from_cells * stride_cell;
++        const size_t sz  = (size_t) (to_cells - from_cells) * stride_cell;
++        // posix_madvise wants the address — work in absolute terms.
++        void * addr = (char *) base + off;
++        // Round up to a page boundary on the low side, round down on the
++        // high side, so we only decommit pages we fully own. The cell-aligned
++        // address is unlikely to be page-aligned in general.
++        const size_t page = (size_t) sysconf(_SC_PAGESIZE);
++        if (page == 0 || page == (size_t) -1) {
++            return;
++        }
++        uintptr_t lo = (uintptr_t) addr;
++        uintptr_t hi = lo + sz;
++        uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1);
++        uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1);
++        if (hi_aligned <= lo_aligned) {
++            continue;
++        }
++        // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with
++        // errno set). We treat any failure as advisory — the RSS win may
++        // be skipped, but correctness is unaffected (the K/V buffer is
++        // still mapped, just possibly still resident).
++        (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned),
++                       MADV_DONTNEED);
++    }
++}
++
++void llama_kv_cache::shrink_if_idle() {
++    if (slot_shrink_idle_ms_ == 0) {
++        return;  // shrink-on-idle disabled
++    }
++    if (kv_size_alloc <= kv_size_shrink_target) {
++        return;  // already at (or below) target — nothing to shrink
++    }
++    if (last_active_ms_ == 0) {
++        return;  // no apply_ubatch has ever run — leave the initial cap as-is
++    }
++
++    const uint64_t now = (uint64_t)
++        std::chrono::duration_cast<std::chrono::milliseconds>(
++            std::chrono::steady_clock::now().time_since_epoch()).count();
++    if (now < last_active_ms_) {
++        return;  // clock went backwards; refuse to act
++    }
++    if ((now - last_active_ms_) < (uint64_t) slot_shrink_idle_ms_) {
++        return;  // not idle long enough
++    }
++
++    const uint32_t from_cells = kv_size_shrink_target;
++    const uint32_t to_cells   = kv_size_alloc;
++
++    // Evict prompt-cache cell metadata for indices [from_cells, to_cells)
++    // on every stream BEFORE decommitting the pages. The server's prompt
++    // cache (server-context.cpp) keeps cells populated between requests so
++    // a subsequent request with the same prefix can skip re-prefill. If we
++    // decommitted pages without invalidating cell metadata, the server's
++    // next prompt-cache lookup could "match" a prefix whose K/V data is
++    // now decommitted (returns to zero pages on next access) = corruption.
++    //
++    // Semantics: shrink-on-idle is allowed to evict the over-cap portion
++    // of the prompt cache because that is exactly what the user opted into
++    // by setting --slot-shrink-idle-ms > 0. A cache miss on a subsequent
++    // long prompt is the cost; the alternative (cap stays grown forever)
++    // defeats Phase 3 entirely — slots that retain cells between requests
++    // (the default for embed + completion) would never shrink.
++    uint32_t evicted = 0;
++    for (uint32_t st = 0; st < n_stream; ++st) {
++        auto & cells = v_cells[st];
++        const uint32_t hi = std::min<uint32_t>(cells.size(), to_cells);
++        for (uint32_t i = from_cells; i < hi; ++i) {
++            if (!cells.is_empty(i)) {
++                cells.rm(i);
++                ++evicted;
++            }
++        }
++    }
++
++    LLAMA_LOG_INFO("%s: shrink soft cap %u -> %u (idle %llu ms, evicted %u cells)\n",
++            __func__, to_cells, from_cells,
++            (unsigned long long) (now - last_active_ms_), evicted);
++
++    for (const auto & layer : layers) {
++        opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells);
++        opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells);
++    }
++
++    // After decommit, treat cells [from_cells, to_cells) as "uncleared" —
++    // their pages will fault back in as fresh zero-filled pages on next
++    // touch, but n_cells_cleared still recorded them as cleared. Roll back
++    // the cleared mark so a future ensure_cleared() re-zeros them only when
++    // they're actually needed.
++    if (n_cells_cleared > from_cells) {
++        n_cells_cleared = from_cells;
++    }
++    kv_size_alloc = from_cells;
++
++    // Refresh last_active_ms_ so we don't immediately try again on the very
++    // next init_batch (which would walk the empty layers for nothing).
++    last_active_ms_ = now;
++}
++
+ void llama_kv_cache::clear(bool data) {
+     for (uint32_t s = 0; s < n_stream; ++s) {
+         v_cells[s].reset();
+@@ -757,6 +938,12 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
+             bool embd_all) {
+     GGML_UNUSED(embd_all);
+ 
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Check idle-shrink BEFORE ensure_cleared(): if shrink runs, it lowers
++    // kv_size_alloc and rolls n_cells_cleared back, so ensure_cleared's
++    // soft-cap-bounded clear targets only the (now smaller) initial cap.
++    shrink_if_idle();
++
+     // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+     ensure_cleared();
+ 
+@@ -804,6 +991,12 @@ llama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool
+ }
+ 
+ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ubatch> & ubatches) {
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Same reason ensure_cleared() is called here as well as init_batch:
++    // the iswa wrapper bypasses init_batch and calls prepare() directly on
++    // each inner kv_cache. shrink_if_idle is idempotent and cheap.
++    shrink_if_idle();
++
+     // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md
+     // The iswa wrapper calls prepare() directly on each inner kv_cache without
+     // going through init_batch(), so the ensure_cleared() in init_batch alone
+@@ -1184,6 +1377,13 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch,
+ }
+ 
+ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch) {
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Refresh the idle timestamp on every actual write. shrink_if_idle uses
++    // this to gate the shrink path.
++    last_active_ms_ = (uint64_t)
++        std::chrono::duration_cast<std::chrono::milliseconds>(
++            std::chrono::steady_clock::now().time_since_epoch()).count();
++
+     // keep track of the max sequence position that we would overwrite with this ubatch
+     // for non-SWA cache, this would be always empty
+     llama_seq_id seq_pos_max_rm[LLAMA_MAX_SEQ];
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -106,6 +106,11 @@ public:
+                      // (Phase 1 behavior). Otherwise clamped to [n_pad, kv_size]
+                      // and rounded up to the next n_pad multiple.
+                      uint32_t   kv_size_initial,
++                     // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++                     // Idle threshold (ms) after which the soft cap is shrunk
++                     // back to kv_size_initial and the over-cap pages are
++                     // decommitted. 0 disables shrink-on-idle.
++                     uint32_t   slot_shrink_idle_ms,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_pad,
+                      uint32_t   n_swa,
+@@ -180,6 +185,17 @@ public:
+     // The current soft cap (cells visible to find_slot).
+     uint32_t get_kv_size_alloc() const { return kv_size_alloc; }
+ 
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // If the soft cap has been grown above kv_size_shrink_target and at least
++    // slot_shrink_idle_ms_ have elapsed since the last apply_ubatch, reset
++    // the soft cap to kv_size_shrink_target and posix_madvise(POSIX_MADV_DONTNEED)
++    // the over-cap pages of every layer's K/V tensors. Idempotent and cheap
++    // when the soft cap is already at the target or shrink-on-idle is disabled.
++    // Called at the top of init_batch and prepare; safe because both happen
++    // before any slot work and the embedder's per-request cells.reset() means
++    // no slot ever holds onto cells across requests.
++    void shrink_if_idle();
++
+     ggml_type type_k() const;
+     ggml_type type_v() const;
+ 
+@@ -296,11 +312,21 @@ private:
+     // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+     // kv_size_max:   hard cap on cells (set from constructor's kv_size).
+     // kv_size_alloc: current soft cap. find_slot only scans cells [0, kv_size_alloc).
+-    //                Grows via ensure_capacity, never shrinks (Phase 2; Phase 3
+-    //                adds shrink-on-idle).
++    //                Grows via ensure_capacity. Phase 3 adds shrink-on-idle:
++    //                shrink_if_idle() may reset it back to kv_size_shrink_target.
+     uint32_t kv_size_max   = 0;
+     uint32_t kv_size_alloc = 0;
+ 
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Phase 3 shrink-on-idle state. kv_size_shrink_target is the cap value
++    // the soft cap is reset to on shrink — set once at ctor from
++    // kv_size_initial (or kv_size_max if Phase-1-mode). slot_shrink_idle_ms_
++    // is the idle threshold in ms; 0 disables shrink. last_active_ms_ is a
++    // monotonic timestamp updated by apply_ubatch.
++    uint32_t kv_size_shrink_target = 0;
++    uint32_t slot_shrink_idle_ms_  = 0;
++    uint64_t last_active_ms_       = 0;
++
+     // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
+     // note: this is not part of the KV state and it's only used to speed-up the find_slot() method
+     std::vector<uint32_t> v_heads;
+diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp
++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+@@ -44,6 +44,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
+         // Hybrid-iswa does not yet thread slot_initial_ctx; pass 0 for
+         // Phase 1 backward-compat (full eager allocation).
+         0,
++        // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++        // Hybrid-iswa does not yet thread slot_shrink_idle_ms; pass 0 for
++        // Phase 2 backward-compat (no shrink-on-idle).
++        0,
+         n_seq_max,
+         n_ubatch,
+         n_pad,
+diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp
+--- a/llama.cpp/src/llama-memory-hybrid.cpp
++++ b/llama.cpp/src/llama-memory-hybrid.cpp
+@@ -43,6 +43,10 @@ llama_memory_hybrid::llama_memory_hybrid(
+         // Hybrid memory does not yet thread slot_initial_ctx; pass 0 for
+         // Phase 1 backward-compat (full eager allocation).
+         0,
++        // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++        // Hybrid memory does not yet thread slot_shrink_idle_ms; pass 0 for
++        // Phase 2 backward-compat (no shrink-on-idle).
++        0,
+         n_seq_max,
+         n_pad,
+         n_swa,
+diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp
+--- a/llama.cpp/src/llama-model.cpp
++++ b/llama.cpp/src/llama-model.cpp
+@@ -2075,6 +2075,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 cparams.n_ctx_seq,
+                                 // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+                                 cparams.slot_initial_ctx,
++                                // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++                                cparams.slot_shrink_idle_ms,
+                                 cparams.n_seq_max,
+                                 cparams.n_ubatch,
+                                 1,
+@@ -2093,6 +2095,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 cparams.n_ctx_seq,
+                                 // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
+                                 cparams.slot_initial_ctx,
++                                // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++                                cparams.slot_shrink_idle_ms,
+                                 cparams.n_seq_max,
+                                 1,
+                                 hparams.n_swa,

patches/0006-kv-reuse-prefix.patch

@@ -0,0 +1,135 @@
+From: opencoti
+Subject: [PATCH 0006] kv-cache reuse for agentic turns — session→slot affinity
+
+opencoti routes an agentic loop through the bundled llamafile server: each
+turn N+1 re-sends turn N's context prefix (system prompt + tool defs +
+history) verbatim, then appends. The server already reuses KV for a matching
+prefix — cache_prompt defaults to true and get_available_slot() picks the
+idle slot whose cached token-prefix best matches the incoming prompt (above
+--slot-prompt-similarity, then LRU). That covers single-session turn-2 reuse
+at the default --parallel 1.
+
+What upstream lacks is *session affinity*. With --parallel N and several
+concurrent opencode sessions sharing slots, the LCP/LRU picker can route a
+session to a slot holding a different session's KV, evicting a prefix that
+would otherwise be reused. This patch adds an optional `session_id` request
+field that keys slot selection:
+
+  - get_available_slot(): if the request carries a non-empty session_id that
+    already maps to an idle slot, return that slot (its resident KV is this
+    session's prefix by construction). The existing LCP→LRU path is the
+    fallback.
+  - launch_slot_with_task(): record session_id -> slot.id when a slot is
+    committed to a session, dropping any stale session previously bound to
+    that slot.
+
+The prefix match itself is still the server's existing token-LCP — affinity
+only ensures the right slot (the one holding the session's KV) is chosen, so
+"(session_id, prefix) reuse" falls out without a separate prefix hash.
+
+Regression shield: empty session_id (the default, and what every
+non-opencoti client sends) skips the affinity phase entirely, leaving slot
+selection byte-for-byte identical to upstream. Builds clean under cosmocc
+(std::map / std::string already available via server-task.h).
+
+Bench: perf/llamafile/turn-2-latency.bench.ts
+Milestone: F5 M0 — see docs/features/advanced_kv.md
+Design: docs/features/advanced_kv.md (M0)
+Upstreaming: deferred; candidate to propose upstream as a generic session-id
+  slot-affinity hint once benched on opencoti workloads.
+---
+diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp
+--- a/llama.cpp/tools/server/server-context.cpp
++++ b/llama.cpp/tools/server/server-context.cpp
+@@ -698,6 +698,13 @@ private:
+     // Necessary similarity of prompt for slot selection
+     float slot_prompt_similarity = 0.0f;
+ 
++    // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
++    // session_id -> slot.id affinity, so an agentic session routes back to the
++    // slot holding its resident KV prefix (turn-to-turn reuse, and no
++    // cross-session eviction when --parallel > 1). Empty session_id is never
++    // inserted, so non-opencoti traffic never consults or mutates this map.
++    std::map<std::string, int> session_to_slot;
++
+     std::string model_name; // name of the loaded model, to be used by API
+     std::set<std::string> model_aliases; // additional names for the model
+     std::set<std::string> model_tags;    // informational tags
+@@ -1249,6 +1256,25 @@ private:
+ 
+         bool update_cache = false;
+ 
++        // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
++        // Session affinity: if this session already owns an idle slot, reuse
++        // it. That slot's resident KV is the session's prefix, so the LCP path
++        // below would pick it anyway when free — but the explicit pin keeps a
++        // concurrent session from stealing/evicting it under --parallel > 1.
++        // Skipped entirely for empty session_id, so upstream selection (LCP
++        // then LRU, below) is unchanged for every non-opencoti request.
++        if (ret == nullptr && !task.params.session_id.empty()) {
++            auto it = session_to_slot.find(task.params.session_id);
++            if (it != session_to_slot.end()) {
++                for (server_slot & slot : slots) {
++                    if (slot.id == it->second && !slot.is_processing()) {
++                        ret = &slot;
++                        break;
++                    }
++                }
++            }
++        }
++
+         // find the slot that has at least n% prompt similarity
+         if (ret == nullptr && slot_prompt_similarity != 0.0f) {
+             float sim_best = 0;
+@@ -1498,6 +1524,22 @@ private:
+             slot.smpl.reset();
+         }
+ 
++        // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
++        // Record session→slot affinity before `task` is moved into the slot,
++        // so this session's next turn routes back here and reuses its KV. A
++        // slot serves one session at a time, so first drop any stale session
++        // that previously mapped to this slot id.
++        if (!task.params.session_id.empty()) {
++            for (auto it = session_to_slot.begin(); it != session_to_slot.end(); ) {
++                if (it->second == slot.id && it->first != task.params.session_id) {
++                    it = session_to_slot.erase(it);
++                } else {
++                    ++it;
++                }
++            }
++            session_to_slot[task.params.session_id] = slot.id;
++        }
++
+         slot.task = std::make_unique<const server_task>(std::move(task));
+ 
+         slot.state = slot.task->is_child()
+diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp
+--- a/llama.cpp/tools/server/server-task.cpp
++++ b/llama.cpp/tools/server/server-task.cpp
+@@ -272,6 +272,7 @@ task_params server_task::params_from_json_cmpl(
+     params.n_discard        = std::max(0, params.n_discard);
+     params.n_cmpl           = json_value(data,       "n_cmpl",             json_value(data, "n", 1));
+     params.n_cache_reuse    = json_value(data,       "n_cache_reuse",      defaults.n_cache_reuse);
++    params.session_id       = json_value(data,       "session_id",         std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
+     //params.t_max_prompt_ms  = json_value(data,       "t_max_prompt_ms",    defaults.t_max_prompt_ms); // TODO: implement
+     params.t_max_predict_ms = json_value(data,       "t_max_predict_ms",   defaults.t_max_predict_ms);
+     params.response_fields  = json_value(data,       "response_fields",    std::vector<std::string>());
+diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h
+--- a/llama.cpp/tools/server/server-task.h
++++ b/llama.cpp/tools/server/server-task.h
+@@ -61,6 +61,14 @@ struct task_params {
+ 
+     int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled)
+ 
++    // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
++    // Optional session affinity key. When non-empty, the server routes this
++    // request back to the slot the same session last used, reusing its
++    // resident KV prefix across agentic turns. Empty (the default, and what
++    // every non-opencoti client sends) preserves upstream slot selection
++    // byte-for-byte.
++    std::string session_id;
++
+     // number of prompt tokens before the latest user message
+     int32_t n_before_user = -1;
+ 

patches/0007-build-header-deps.patch

@@ -0,0 +1,92 @@
+From: opencoti
+Subject: [PATCH 0007] build infra — compiler-driven header dependency tracking
+
+Editing any llama.cpp/src (or common/, tools/server/) header never marked its
+dependent objects out of date, so every header change forced a full
+from-scratch rebuild — the single biggest drag on iterating the F4/F5 vendored
+patches.
+
+Root cause: the build tracks header deps via cosmopolitan's `mkdeps`
+(build/deps.mk → o/$(MODE)/depend). mkdeps only resolves cosmopolitan's
+root-relative include style; llama.cpp's sources use `#include "foo.h"`
+resolved through `-iquote` dirs, which mkdeps cannot follow, so it emitted
+ZERO edges for llama.cpp/src headers. Source->object tracking worked (the
+pattern rule), but header->object did not.
+
+Fix (compiler-driven deps, the modern standard): add `-MMD` to the COMPILE.c /
+COMPILE.cc commands so cosmocc emits an accurate per-translation-unit
+`<obj>.o.d` (project headers only — system headers excluded), then `-include`
+those `.o.d` files in build/deps.mk. clang resolves the real `-iquote`
+includes, so the header set is exact. cosmocc supports -MMD natively
+(get_dependency_outputs in bin/cosmocc): the x86 `.o.d` lands next to the
+object and an `.aarch64/*.o.d` twin under the arch subdir; only the x86 copy is
+included (header sets are arch-independent).
+
+Effect: a header edit now recompiles exactly the affected objects (both fat
+arches, via the existing rule), and nothing else — no more scratch rebuilds.
+The first build after this patch lands populates the .o.d set (a one-time full
+compile); every build thereafter is correctly incremental.
+
+Interaction with ccache: ccache wraps cosmocc's FAT driver, so a cache hit
+restores only the x86 .o (never the .aarch64 twin). With accurate header
+tracking, objects are no longer manually deleted to force recompiles, so the
+.aarch64 twin always persists on disk in a warm tree and hits stay safe. The
+remaining hazard — a warm cache restoring x86-only objects into a freshly
+reset (o/-wiped) tree — is handled in the opencoti build pipeline, which runs
+the full `build` with CCACHE_RECACHE=1.
+
+Bench: docs/features/llamafile_build.md (incremental-rebuild verification:
+touch an included header -> only its dependents recompile; touch a
+non-dependency header -> no recompile)
+Milestone: build infrastructure (supports F4/F5 — see docs/features/advanced_kv.md)
+Upstreaming: build-only; no behavior change to the produced binary. Not
+intended for upstream (cosmopolitan/llamafile use mkdeps by design); opencoti
+carries it to make iterating the vendored patch series tractable.
+
+diff --git a/build/deps.mk b/build/deps.mk
+--- a/build/deps.mk
++++ b/build/deps.mk
+@@ -23,3 +23,15 @@ $(INCS):
+ 	rm -f o/$(MODE)/depend
+ 
+ -include o/$(MODE)/depend
++
++# opencoti build-header-deps — see docs/features/llamafile_build.md
++# Pull in the per-object dependency files cosmocc emits under -MMD (added to
++# build/rules.mk COMPILE.*). mkdeps' o/$(MODE)/depend above cannot resolve
++# llama.cpp/src's -iquote includes, so it tracked zero of those headers and a
++# header edit never recompiled its dependents — every change forced a
++# from-scratch rebuild. The compiler-emitted `<obj>.o.d` files carry the exact
++# header set per translation unit. Only the x86 `.o.d` is pulled in; the
++# `.aarch64/*.o.d` twins list identical headers (arch-independent). The shell
++# find runs at parse time over prior-build outputs (none on a cold build → a
++# harmless empty include, everything compiles anyway).
++-include $(shell [ -d o/$(MODE) ] && find o/$(MODE) -name '*.o.d' -not -path '*/.aarch64/*' 2>/dev/null)
+diff --git a/build/rules.mk b/build/rules.mk
+--- a/build/rules.mk
++++ b/build/rules.mk
+@@ -6,8 +6,22 @@
+ # ==============================================================================
+ 
+ LINK.o = $(CXX) $(CCFLAGS) $(LDFLAGS)
+-COMPILE.c = $(CC) $(CCFLAGS) $(CFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -c
+-COMPILE.cc = $(CXX) $(CCFLAGS) $(CXXFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -c
++# opencoti build-header-deps — see docs/features/llamafile_build.md
++# -MMD makes cosmocc emit a per-TU dependency file listing the real (project,
++# not system) header set; build/deps.mk -includes them so a header edit
++# recompiles exactly the affected objects. Without this, mkdeps' depend tracked
++# zero llama.cpp/src headers (it can't resolve -iquote includes), so any header
++# change forced a from-scratch rebuild.
++#
++# `-MF $@.d` is REQUIRED, not cosmetic: cosmocc defaults the dep file to
++# `<obj>.d` (it appends `.d` → e.g. foo.cpp.o.d), but ccache derives the
++# expected name from `-o foo.cpp.o` by stripping `.o` → foo.cpp.d. That
++# mismatch made ccache stat a file cosmocc never wrote ("failed to stat
++# …foo.cpp.d") and abort every compile with "ccache internal error" → nothing
++# was ever cached. Pinning -MF to cosmocc's actual name (`$@.d` = foo.cpp.o.d)
++# makes the two agree, so ccache caches and restores normally.
++COMPILE.c = $(CC) $(CCFLAGS) $(CFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -MMD -MF $@.d -c
++COMPILE.cc = $(CXX) $(CCFLAGS) $(CXXFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -MMD -MF $@.d -c
+ 
+ # ==============================================================================
+ # Standard Compilation Rules

patches/0010-rest-kv-eviction.patch

@@ -0,0 +1,476 @@
+From: opencoti
+Subject: [PATCH 0010] retention-aware KV eviction — KeyDiff key-similarity, server-window
+
+When an agentic context exceeds a slot's n_ctx, the server's context-shift
+discards the *positional middle* [n_keep, n_keep+n_discard) regardless of what
+sits there. This patch makes that choice retention-aware: it discards the
+lowest-value *contiguous window* of the same width instead, keeping the
+distinctive tokens and dropping the redundant ones.
+
+Reference choice. The named technique (TRIM-KV, arXiv 2605.09649) needs
+per-model retention gates trained by distillation — not a drop-in runtime
+patch. Attention-score methods (H2O, SnapKV) need the materialized attention
+matrix, which Flash Attention never produces (the vendored pin defaults
+flash_attn_type = AUTO). So this substitutes a score-free, FA-compatible
+criterion — KeyDiff (arXiv 2504.15364): score each cached key by its
+*distinctiveness*, the mean over heads of (1 - cosine(key_head, mean_head)).
+Keys close to the sequence's mean are redundant (evict); keys far from it are
+distinctive (keep).
+
+Mechanism (the "server-window" depth — mirror-safe):
+  - src/llama-memory.h: a new non-pure virtual seq_key_scores(seq_id,
+    layer_hint) with a default empty body, so no backend is forced to implement
+    it and the pure-virtual surface is unchanged. (Adding it grows the vtable —
+    a consistent recompile of all llama.cpp TUs is required; see the build
+    gotcha below.)
+  - src/llama-kv-cache.cpp: the real implementation — read each cached key of
+    one representative layer to host (amortized: only at the rare context-shift
+    trigger, unlike per-decode-interval scoring), group into heads, return a
+    per-position distinctiveness vector (+INF for absent positions). Empty when
+    unsupported (no cells, or a key type with no float converter) so the caller
+    falls back to positional discard.
+  - src/llama-kv-cache-iswa.cpp: delegate to the non-SWA base cache.
+  - src/llama-context.cpp + include/llama.h: a thin C shim
+    llama_memory_seq_key_scores() — a plain virtual call (no RTTI/downcast).
+  - tools/server/server-context.cpp: inside the EXISTING context-shift block,
+    when rest_kv_eviction is on, slide a width-n_discard window over
+    [n_keep, n_tokens - rest_kv_recent) and discard the minimum-retention one.
+    It is still ONE contiguous window, so the seq_rm/seq_add + token-mirror
+    rewrite stay 1:1 with KV positions — no rework of prefix-reuse/sampler.
+  - common/common.h + common/arg.cpp: --rest-kv-eviction (off by default),
+    --rest-kv-recent N (protect the recent tail), --rest-kv-layer N (-1 = auto).
+
+Regression shield: --rest-kv-eviction off (the default, and every non-opencoti
+client) leaves w0 = n_keep, i.e. context-shift is byte-for-byte upstream. Any
+failure (non-KV memory, quantized keys, no room after the recent tail) also
+falls back to the positional middle.
+
+Build gotcha (buglog bug-175): seq_key_scores is a new virtual on
+llama_memory_i, so the vtable layout changes. The llamafile Makefile tracks no
+header deps; force a consistent recompile of all llama.cpp src/common/tools
+objects (delete x86 .o, rebuild) — and because cosmocc is a fat compiler whose
+aarch64 sibling object is NOT restored by a ccache hit, delete only x86 .o that
+lack an aarch64 twin and rebuild with CCACHE_DISABLE=1, or use the full
+reproducible build:llamafile.
+
+Bench: perf/llamafile/rest-kv-eviction.bench.ts
+Milestone: F5 M1 — see docs/features/advanced_kv.md
+Design: docs/features/advanced_kv.md (M1)
+Upstreaming: deferred; candidate to propose upstream as a pluggable
+  context-shift eviction policy once benched on opencoti workloads.
+---
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1398,6 +1398,29 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.ctx_shift = value;
+         }
+     ).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY}).set_env("LLAMA_ARG_CONTEXT_SHIFT"));
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--rest-kv-eviction"},
++        {"--no-rest-kv-eviction"},
++        string_format("when context shift fires, evict the lowest-retention (KeyDiff key-similarity) contiguous window instead of the positional middle (default: %s)", params.rest_kv_eviction ? "enabled" : "disabled"),
++        [](common_params & params, bool value) {
++            params.rest_kv_eviction = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_EVICTION"));
++    add_opt(common_arg(
++        {"--rest-kv-recent"}, "N",
++        string_format("rest-kv: protect the most-recent N tokens from eviction (default: %d)", params.rest_kv_recent),
++        [](common_params & params, int value) {
++            params.rest_kv_recent = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_RECENT"));
++    add_opt(common_arg(
++        {"--rest-kv-layer"}, "N",
++        string_format("rest-kv: model layer whose keys score retention, -1 = auto/middle (default: %d)", params.rest_kv_layer),
++        [](common_params & params, int value) {
++            params.rest_kv_layer = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER"));
+     add_opt(common_arg(
+         {"--chunks"}, "N",
+         string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -554,6 +554,14 @@ struct common_params {
+     bool swa_full          = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
+     bool kv_unified        = false; // enable unified KV cache
+ 
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    // When a slot fills n_ctx and ctx_shift fires, discard the lowest-retention
++    // *contiguous* window (KeyDiff key-similarity) instead of the blind
++    // positional middle. Off => byte-for-byte upstream context-shift.
++    bool    rest_kv_eviction = false; // master switch (default off)
++    int32_t rest_kv_recent   = 256;   // protect the most-recent N tokens from eviction
++    int32_t rest_kv_layer    = -1;    // representative layer for scoring; -1 = auto (middle)
++
+     bool input_prefix_bos  = false; // prefix BOS to user inputs, preceding input_prefix
+     bool use_mmap          = true;  // enable mmap to use filesystem cache
+     bool use_direct_io     = false; // read from disk without buffering
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -730,6 +730,19 @@ extern "C" {
+                  llama_pos p0,
+                  llama_pos p1);
+ 
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    // Fill out[0..n) with per-position KeyDiff retention scores for seq_id
++    // (higher = keep, +INF = absent position). layer_hint < 0 selects a default
++    // representative layer. Returns the number of finite scores written; 0 when
++    // unsupported (non-KV memory, a key type with no float converter, or an
++    // empty sequence) — the caller should then fall back to positional eviction.
++    LLAMA_API size_t llama_memory_seq_key_scores(
++            llama_memory_t mem,
++              llama_seq_id seq_id,
++                   int32_t layer_hint,
++                     float * out,
++                    size_t   n);
++
+     // Copy all tokens that belong to the specified sequence to another sequence
+     // p0 < 0 : [0,  p1]
+     // p1 < 0 : [p0, inf)
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -3732,6 +3732,38 @@ bool llama_memory_seq_rm(
+     return mem->seq_rm(seq_id, p0, p1);
+ }
+ 
++// opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++// Thin C shim over the (defaulted) virtual llama_memory_i::seq_key_scores — a
++// plain virtual dispatch, so no RTTI/downcast and no change to the pure-virtual
++// surface of the interface. Non-KV memories inherit the empty default and this
++// returns 0, leaving the server on its positional fallback.
++size_t llama_memory_seq_key_scores(
++        llama_memory_t mem,
++          llama_seq_id seq_id,
++               int32_t layer_hint,
++                 float * out,
++                size_t   n) {
++    if (!mem || !out || n == 0) {
++        return 0;
++    }
++
++    const std::vector<float> scores = mem->seq_key_scores(seq_id, layer_hint);
++    if (scores.empty()) {
++        return 0;
++    }
++
++    size_t cnt = 0;
++    for (size_t p = 0; p < n; ++p) {
++        const float s = p < scores.size() ? scores[p] : std::numeric_limits<float>::infinity();
++        out[p] = s;
++        if (std::isfinite(s)) {
++            cnt++;
++        }
++    }
++
++    return cnt;
++}
++
+ void llama_memory_seq_cp(
+         llama_memory_t mem,
+           llama_seq_id seq_id_src,
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp
+--- a/llama.cpp/src/llama-kv-cache-iswa.cpp
++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp
+@@ -117,6 +117,14 @@ llama_pos llama_kv_cache_iswa::seq_pos_max(llama_seq_id seq_id) const {
+     return kv_swa->seq_pos_max(seq_id);
+ }
+ 
++std::vector<float> llama_kv_cache_iswa::seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const {
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    // Score on the non-SWA base cache: the SWA cache only ever retains a
++    // sliding window, so its keys are not the right eviction target. The base
++    // cache is a superset of the SWA cache, so it has every position we score.
++    return kv_base->seq_key_scores(seq_id, layer_hint);
++}
++
+ std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdown() const {
+     std::map<ggml_backend_buffer_type_t, size_t> mb = kv_base->memory_breakdown();
+     for (const auto & buft_size : kv_swa->memory_breakdown()) {
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h
+--- a/llama.cpp/src/llama-kv-cache-iswa.h
++++ b/llama.cpp/src/llama-kv-cache-iswa.h
+@@ -62,6 +62,9 @@ public:
+     llama_pos seq_pos_min(llama_seq_id seq_id) const override;
+     llama_pos seq_pos_max(llama_seq_id seq_id) const override;
+ 
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    std::vector<float> seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const override;
++
+     std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
+ 
+     // state write/load
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -711,6 +711,137 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
+     return true;
+ }
+ 
++std::vector<float> llama_kv_cache::seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const {
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    // KeyDiff-style, score-free (Flash-Attention-compatible) retention signal:
++    // for one representative layer, read every cached key of `seq_id`, group
++    // it into attention heads, and score each position by its key's
++    // *distinctiveness* — the mean over heads of (1 - cosine(key_head,
++    // mean_head)). High = far from the sequence's mean key = keep; low =
++    // redundant = safe to evict. The server uses this to choose the
++    // lowest-retention contiguous window to discard during context-shift,
++    // instead of the blind positional middle. Returns scores indexed by token
++    // position, +INF for absent positions (never selected). An empty result
++    // means "unsupported here" (no cells, or keys in a type with no float
++    // converter) and the caller falls back to positional discard. Reads happen
++    // only at the (infrequent) context-shift trigger, so the device->host copy
++    // is amortized — unlike per-decode-interval scoring.
++
++    if (seq_id < 0 || (size_t) seq_id >= seq_to_stream.size() || layers.empty()) {
++        return {};
++    }
++
++    const uint32_t strm  = seq_to_stream[seq_id];
++    const auto &   cells = v_cells[strm];
++
++    const llama_pos pmax = cells.seq_pos_max(seq_id);
++    if (pmax < 0) {
++        return {};
++    }
++
++    // pick a representative cache layer. A non-negative hint first tries to map
++    // a *model* layer id into the cache; failing that it is clamped as a direct
++    // cache-layer index. Default (hint < 0) is the middle layer — empirically
++    // the most informative for retention and cheap (one layer's keys only).
++    int32_t cil = (int32_t) layers.size() / 2;
++    if (layer_hint >= 0) {
++        const auto it = map_layer_ids.find(layer_hint);
++        cil = it != map_layer_ids.end()
++            ? it->second
++            : std::min<int32_t>(layer_hint, (int32_t) layers.size() - 1);
++    }
++
++    const auto &   layer = layers[cil];
++    const uint32_t mil   = layer.il;
++
++    const uint32_t n_head_kv     = hparams.n_head_kv(mil);
++    const uint32_t n_embd_head_k = hparams.n_embd_head_k(mil);
++    const uint32_t n_embd_k_gqa  = n_head_kv * n_embd_head_k;
++    if (n_head_kv == 0 || n_embd_head_k == 0 || n_embd_k_gqa == 0) {
++        return {};
++    }
++
++    ggml_tensor * k = layer.k_stream[strm];
++    if (!k) {
++        return {};
++    }
++
++    const auto * tt = ggml_get_type_traits(k->type);
++    if (!tt || !tt->to_float) {
++        return {}; // e.g. a quantized K cache with no dequantizer -> positional fallback
++    }
++
++    const size_t row_bytes = ggml_row_size(k->type, n_embd_k_gqa);
++
++    // collect (pos, cell index) for this sequence, ascending by position
++    std::vector<std::pair<llama_pos, uint32_t>> pc;
++    pc.reserve(cells.get_used());
++    for (uint32_t i = 0; i < cells.size(); ++i) {
++        if (!cells.is_empty(i) && cells.seq_has(i, seq_id)) {
++            pc.emplace_back(cells.pos_get(i), i);
++        }
++    }
++    if (pc.empty()) {
++        return {};
++    }
++    std::sort(pc.begin(), pc.end());
++
++    const size_t n = pc.size();
++
++    // read each key row once (amortized device->host copy), dequantized to f32
++    std::vector<float>   keys((size_t) n * n_embd_k_gqa);
++    std::vector<uint8_t> rowbuf(row_bytes);
++    for (size_t j = 0; j < n; ++j) {
++        const uint32_t i = pc[j].second;
++        ggml_backend_tensor_get(k, rowbuf.data(), (size_t) i * row_bytes, row_bytes);
++        tt->to_float(rowbuf.data(), keys.data() + j * n_embd_k_gqa, (int64_t) n_embd_k_gqa);
++    }
++
++    // per-head mean key across the sequence, then per-head mean norm
++    std::vector<float> mean((size_t) n_embd_k_gqa, 0.0f);
++    for (size_t j = 0; j < n; ++j) {
++        const float * kv = keys.data() + j * n_embd_k_gqa;
++        for (uint32_t e = 0; e < n_embd_k_gqa; ++e) {
++            mean[e] += kv[e];
++        }
++    }
++    for (uint32_t e = 0; e < n_embd_k_gqa; ++e) {
++        mean[e] /= (float) n;
++    }
++
++    std::vector<float> mean_norm(n_head_kv, 0.0f);
++    for (uint32_t h = 0; h < n_head_kv; ++h) {
++        const float * mh = mean.data() + (size_t) h * n_embd_head_k;
++        double s = 0.0;
++        for (uint32_t d = 0; d < n_embd_head_k; ++d) {
++            s += (double) mh[d] * (double) mh[d];
++        }
++        mean_norm[h] = (float) std::sqrt(s);
++    }
++
++    // distinctiveness per position = mean over heads of (1 - cos(key_head, mean_head))
++    std::vector<float> out((size_t) pmax + 1, std::numeric_limits<float>::infinity());
++    for (size_t j = 0; j < n; ++j) {
++        const float * kv = keys.data() + j * n_embd_k_gqa;
++        double acc = 0.0;
++        for (uint32_t h = 0; h < n_head_kv; ++h) {
++            const float * kh = kv          + (size_t) h * n_embd_head_k;
++            const float * mh = mean.data() + (size_t) h * n_embd_head_k;
++            double dot = 0.0, kn = 0.0;
++            for (uint32_t d = 0; d < n_embd_head_k; ++d) {
++                dot += (double) kh[d] * (double) mh[d];
++                kn  += (double) kh[d] * (double) kh[d];
++            }
++            const double denom = std::sqrt(kn) * (double) mean_norm[h];
++            const double cos   = denom > 0.0 ? dot / denom : 0.0;
++            acc += (1.0 - cos);
++        }
++        out[(size_t) pc[j].first] = (float) (acc / (double) n_head_kv);
++    }
++
++    return out;
++}
++
+ void llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
+     GGML_ASSERT(seq_id_src >= 0 && (size_t) seq_id_src < seq_to_stream.size());
+     GGML_ASSERT(seq_id_dst >= 0 && (size_t) seq_id_dst < seq_to_stream.size());
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -146,6 +146,9 @@ public:
+     llama_pos seq_pos_min(llama_seq_id seq_id) const override;
+     llama_pos seq_pos_max(llama_seq_id seq_id) const override;
+ 
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    std::vector<float> seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const override;
++
+     std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
+ 
+     // state write/load
+diff --git a/llama.cpp/src/llama-memory.h b/llama.cpp/src/llama-memory.h
+--- a/llama.cpp/src/llama-memory.h
++++ b/llama.cpp/src/llama-memory.h
+@@ -6,6 +6,7 @@
+ #include <map>
+ #include <memory>
+ #include <functional>
++#include <vector>
+ 
+ struct llama_ubatch;
+ 
+@@ -112,6 +113,22 @@ struct llama_memory_i {
+     virtual llama_pos seq_pos_min(llama_seq_id seq_id) const = 0;
+     virtual llama_pos seq_pos_max(llama_seq_id seq_id) const = 0;
+ 
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    // Per-position KeyDiff "retention" scores for one sequence: higher = more
++    // distinctive (key far from the sequence's mean key) = keep; lower = more
++    // redundant = safe to evict. The server uses these to pick WHICH window to
++    // discard during context-shift, instead of the blind positional middle.
++    // Returns a vector indexed by token position [0, seq_pos_max]; absent
++    // positions are left at +INF (never evicted). The default returns empty so
++    // non-KV memories — and any caller that never opts in — are byte-for-byte
++    // unaffected (the eviction caller then falls back to positional discard).
++    // layer_hint < 0 selects a default representative layer.
++    virtual std::vector<float> seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const {
++        (void) seq_id;
++        (void) layer_hint;
++        return {};
++    }
++
+     virtual std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const = 0;
+ 
+     //
+diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp
+--- a/llama.cpp/tools/server/server-context.cpp
++++ b/llama.cpp/tools/server/server-context.cpp
+@@ -2408,14 +2408,63 @@ private:
+                 const int n_left    = slot.prompt.n_tokens() - n_keep;
+                 const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2);
+ 
++                // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++                // Upstream discards the positional middle [n_keep, n_keep+n_discard).
++                // When rest-kv-eviction is on, slide that same-width window over the
++                // evictable region [n_keep, n_tokens - rest_kv_recent) and discard the
++                // window with the LOWEST retention sum (KeyDiff key-similarity) — keep the
++                // distinctive tokens, drop the redundant ones. It is still ONE contiguous
++                // window, so the seq_rm/seq_add + token-mirror rewrite below stay 1:1 with
++                // KV positions. Any failure (eviction off, non-KV memory, quantized keys,
++                // no room after protecting the recent tail) leaves w0 = n_keep, i.e. the
++                // byte-for-byte upstream positional discard.
++                // opencoti F5 M6 MTP-composition: scoring reads ctx_tgt (the authoritative
++                // full-model KV); the SAME window w0 is then applied to BOTH the target and
++                // the draft KV cache below, preserving target/draft position alignment for
++                // speculative decode.
++                int w0 = n_keep;
++                if (params_base.rest_kv_eviction && n_discard > 0) {
++                    const int n_tokens = slot.prompt.n_tokens();
++                    const int recent   = std::max(0, params_base.rest_kv_recent);
++                    const int hi       = n_tokens - recent - n_discard; // last valid window start (inclusive)
++                    if (hi >= n_keep) {
++                        std::vector<float> scores(n_tokens, std::numeric_limits<float>::infinity());
++                        const size_t got = llama_memory_seq_key_scores(
++                            llama_get_memory(ctx_tgt), slot.id, params_base.rest_kv_layer, scores.data(), n_tokens);
++                        if (got > 0) {
++                            // prefix sums over [n_keep, n_tokens); an absent position gets a
++                            // large finite penalty so a window covering one is never picked
++                            // (avoids inf/inf arithmetic in the sliding sum).
++                            const int m = n_tokens - n_keep;
++                            std::vector<double> pref(m + 1, 0.0);
++                            for (int i = 0; i < m; ++i) {
++                                const float sv = scores[n_keep + i];
++                                pref[i + 1] = pref[i] + (std::isfinite(sv) ? (double) sv : 1e30);
++                            }
++                            double best    = std::numeric_limits<double>::infinity();
++                            int    best_w0 = n_keep;
++                            for (int s = n_keep; s <= hi; ++s) {
++                                const double sum = pref[s - n_keep + n_discard] - pref[s - n_keep];
++                                if (sum < best) {
++                                    best    = sum;
++                                    best_w0 = s;
++                                }
++                            }
++                            w0 = best_w0;
++                            SLT_WRN(slot, "rest-kv eviction: discard [%d, %d) retention=%.3f (vs positional [%d, %d))\n",
++                                    w0, w0 + n_discard, best, n_keep, n_keep + n_discard);
++                        }
++                    }
++                }
++
+                 SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard);
+ 
+-                common_context_seq_rm (ctx_tgt, slot.id, n_keep            , n_keep + n_discard);
+-                common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard);
++                common_context_seq_rm (ctx_tgt, slot.id, w0            , w0 + n_discard);
++                common_context_seq_add(ctx_tgt, slot.id, w0 + n_discard, slot.prompt.n_tokens(), -n_discard);
+ 
+                 if (ctx_dft) {
+-                    common_context_seq_rm (ctx_dft.get(), slot.id, n_keep            , n_keep + n_discard);
+-                    common_context_seq_add(ctx_dft.get(), slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
++                    common_context_seq_rm (ctx_dft.get(), slot.id, w0            , w0 + n_discard);
++                    common_context_seq_add(ctx_dft.get(), slot.id, w0 + n_discard, slot.prompt.tokens.pos_next(), -n_discard);
+                 }
+ 
+                 // add generated tokens to cache
+@@ -2424,7 +2473,7 @@ private:
+                     GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
+ 
+                     llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy
+-                    for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) {
++                    for (size_t i = w0 + n_discard; i < new_tokens.size(); i++) {
+                         new_tokens[i - n_discard] = new_tokens[i];
+                     }
+ 

patches/0020-headinfer-per-head.patch

@@ -0,0 +1,520 @@
+From: opencoti
+Subject: [PATCH 0020] F5 M2 — HeadInfer: head-wise GPU/CPU KV residency split
+
+Keep a configurable fraction of KV heads GPU-resident; offload the rest to
+host memory; reassemble the full head set via ggml_concat on the head axis
+at each attention step. Default off (frac = 1.0). Validated 6/6 on solidPC
+RTX 3090 with Qwen2.5-Coder-0.5B-IQ4_XS at -ngl 99: GPU KV halves (48 → 24
+MiB at frac=0.5), CPU half appears (24 MiB), greedy decode byte-identical
+between baseline and split. See docs/features/advanced_kv.md (F5 M2).
+
+The CUDA backend's supports_op for GGML_OP_CONCAT was over-broad (claimed
+any non-I32/I16 type) while the kernel ggml_cuda_op_concat only implements
+F32. Tightened to match the kernel — F16 concat now correctly routes to the
+CPU backend (which has a real F16 path at ggml-cpu/ops.cpp:1980). Required
+for the reassembly to run on F16 KV caches.
+
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1421,6 +1421,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.rest_kv_layer = value;
+         }
+     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER"));
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--headinfer-gpu-heads-frac"}, "F",
++        string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac),
++        [](common_params & params, const std::string & value) {
++            params.headinfer_gpu_heads_frac = std::stof(value);
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC"));
+     add_opt(common_arg(
+         {"--chunks"}, "N",
+         string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
+diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp
+--- a/llama.cpp/common/common.cpp
++++ b/llama.cpp/common/common.cpp
+@@ -1632,6 +1632,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
+     cparams.slot_initial_ctx  = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx;
+     // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+     cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms;
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
+     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+     cparams.n_batch           = params.n_batch;
+     cparams.n_ubatch          = params.n_ubatch;
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -435,6 +435,11 @@ struct common_params {
+     // slot_initial_ctx and the over-cap pages are decommitted via
+     // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior).
+     int32_t slot_shrink_idle_ms   =     0;
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    // Fraction of KV heads kept GPU-resident; the rest are offloaded to host
++    // memory and streamed back per step. 1.0 = off (no split). Only active for
++    // GPU-offloaded layers with flash-attention (non-transposed V cache).
++    float   headinfer_gpu_heads_frac = 1.0f;
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
+diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+@@ -5308,8 +5308,16 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de
+                 return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15);
+         case GGML_OP_CONCAT:
+             {
+-                ggml_type src0_type = op->src[0]->type;
+-                return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;
++                // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++                // Tighten to F32-only to match ggml_cuda_op_concat's three
++                // hard asserts at ggml-cuda/concat.cu:158-160. The over-broad
++                // upstream filter ("any type except I32/I16") trusts the
++                // scheduler into dispatching F16 concats to CUDA, where the
++                // kernel asserts; with this fix the scheduler correctly routes
++                // F16 concat to the CPU backend, which has a real F16 path
++                // (ggml-cpu/ops.cpp:1980 concat_f16). Required for M2 get_k /
++                // get_v reassembly on F16 KV caches.
++                return op->src[0]->type == GGML_TYPE_F32;
+             } break;
+         case GGML_OP_CONV_TRANSPOSE_1D:
+             {
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -353,6 +353,11 @@ extern "C" {
+         // (Phase 2 behavior — soft cap grows monotonically). Requires
+         // slot_initial_ctx > 0 to have any visible effect.
+         uint32_t slot_shrink_idle_ms;
++        // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++        // Fraction of KV heads kept GPU-resident (rest offloaded to host and
++        // streamed back per step). 1.0 = off. Only effective for offloaded
++        // layers with flash-attention (non-transposed V cache).
++        float    headinfer_gpu_heads_frac;
+         uint32_t n_rs_seq;          // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
+         int32_t  n_threads;         // number of threads to use for generation
+         int32_t  n_threads_batch;   // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -62,6 +62,8 @@ llama_context::llama_context(
+     cparams.slot_initial_ctx = params.slot_initial_ctx;
+     // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+     cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms;
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
+ 
+     cparams.n_threads        = params.n_threads;
+     cparams.n_threads_batch  = params.n_threads_batch;
+@@ -3353,6 +3355,7 @@ llama_context_params llama_context_default_params() {
+         /*.n_seq_max                   =*/ 1,
+         /*.slot_initial_ctx            =*/ 0,
+         /*.slot_shrink_idle_ms         =*/ 0,
++        /*.headinfer_gpu_heads_frac    =*/ 1.0f,
+         /*.n_rs_seq                    =*/ 0,
+         /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+         /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -21,6 +21,8 @@ struct llama_cparams {
+     // milliseconds: when a grown KV-cache is idle for at least this long, the
+     // soft cap is reset to slot_initial_ctx and the pages beyond are decommitted.
+     uint32_t slot_shrink_idle_ms;
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    float    headinfer_gpu_heads_frac;
+     uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+     int32_t  n_threads;       // number of threads to use for generation
+     int32_t  n_threads_batch; // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp
+--- a/llama.cpp/src/llama-kv-cache-iswa.cpp
++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp
+@@ -22,6 +22,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+                  uint32_t   kv_size,
+                  uint32_t   kv_size_initial,
+                  uint32_t   slot_shrink_idle_ms,
++                 float      headinfer_gpu_heads_frac,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_ubatch,
+                  uint32_t   n_pad,
+@@ -64,6 +65,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+     kv_base = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+             v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms,
++            headinfer_gpu_heads_frac,
+             n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse);
+ 
+     LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
+@@ -71,6 +73,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
+     kv_swa = std::make_unique<llama_kv_cache>(
+             model, type_k, type_v,
+             v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms,
++            headinfer_gpu_heads_frac,
+             n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse);
+ }
+ 
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h
+--- a/llama.cpp/src/llama-kv-cache-iswa.h
++++ b/llama.cpp/src/llama-kv-cache-iswa.h
+@@ -28,6 +28,8 @@ public:
+                      // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+                      // Forwarded to both kv_base and kv_swa. 0 = shrink disabled.
+                      uint32_t   slot_shrink_idle_ms,
++                     // opencoti F5 M2 headinfer — forwarded to both kv_base and kv_swa.
++                     float      headinfer_gpu_heads_frac,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_ubatch,
+                      uint32_t   n_pad,
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -110,6 +110,7 @@ llama_kv_cache::llama_kv_cache(
+                  uint32_t   kv_size,
+                  uint32_t   kv_size_initial,
+                  uint32_t   slot_shrink_idle_ms,
++                 float      headinfer_gpu_heads_frac,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_pad,
+                  uint32_t   n_swa,
+@@ -262,23 +263,67 @@ llama_kv_cache::llama_kv_cache(
+         const bool has_k = true;
+         const bool has_v = !is_mla;
+ 
+-        ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gqa, kv_size, n_stream) : nullptr;
+-        ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gqa, kv_size, n_stream) : nullptr;
++        // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++        // Head-residency split: keep the first gpu_heads KV heads on this
++        // (offloaded/device) layer's buffer and put the remaining heads on a
++        // host buffer. Splitting on KV-head boundaries is inherently GQA-safe.
++        // Gated to offloaded, non-transposed-V, unified (n_stream==1) layers;
++        // frac >= 1.0 (or <= 1 head) means no split (byte-identical upstream).
++        uint32_t gpu_heads    = 0;
++        uint32_t n_embd_k_gpu = n_embd_k_gqa;
++        uint32_t n_embd_v_gpu = n_embd_v_gqa;
++        uint32_t n_embd_k_cpu = 0;
++        uint32_t n_embd_v_cpu = 0;
++        if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans && n_stream == 1) {
++            const uint32_t n_head_kv     = hparams.n_head_kv(il);
++            const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++            const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++            if (n_head_kv > 1) {
++                uint32_t g = (uint32_t) std::lround((double) headinfer_gpu_heads_frac * (double) n_head_kv);
++                g = std::max(1u, std::min(g, n_head_kv - 1));
++                gpu_heads    = g;
++                n_embd_k_gpu = g * n_embd_head_k;
++                n_embd_v_gpu = g * n_embd_head_v;
++                n_embd_k_cpu = (n_head_kv - g) * n_embd_head_k;
++                n_embd_v_cpu = (n_head_kv - g) * n_embd_head_v;
++                LLAMA_LOG_INFO("%s: layer %3d: headinfer split heads %u GPU / %u CPU\n",
++                        __func__, il, gpu_heads, n_head_kv - gpu_heads);
++            }
++        }
++
++        ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gpu, kv_size, n_stream) : nullptr;
++        ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gpu, kv_size, n_stream) : nullptr;
+ 
+         has_k && ggml_format_name(k, "cache_k_l%d", il);
+         has_v && ggml_format_name(v, "cache_v_l%d", il);
+ 
++        // opencoti F5 M2 headinfer — the host-resident head subset lives in the
++        // CPU buffer-type context; the buffer-allocation loop below picks it up
++        // automatically (ctx_for_buft keys contexts by buffer type).
++        ggml_tensor * k_cpu = nullptr;
++        ggml_tensor * v_cpu = nullptr;
++        if (gpu_heads > 0) {
++            ggml_context * ctx_cpu = ctx_for_buft(ggml_backend_cpu_buffer_type());
++            if (!ctx_cpu) {
++                throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache");
++            }
++            k_cpu = has_k ? ggml_new_tensor_3d(ctx_cpu, type_k, n_embd_k_cpu, kv_size, n_stream) : nullptr;
++            v_cpu = has_v ? ggml_new_tensor_3d(ctx_cpu, type_v, n_embd_v_cpu, kv_size, n_stream) : nullptr;
++            has_k && k_cpu && ggml_format_name(k_cpu, "cache_k_cpu_l%d", il);
++            has_v && v_cpu && ggml_format_name(v_cpu, "cache_v_cpu_l%d", il);
++        }
++
+         std::vector<ggml_tensor *> k_stream;
+         std::vector<ggml_tensor *> v_stream;
+ 
+         for (uint32_t s = 0; s < n_stream; ++s) {
+-            k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gqa, kv_size, k->nb[1], s*k->nb[2]) : nullptr);
+-            v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
++            k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gpu, kv_size, k->nb[1], s*k->nb[2]) : nullptr);
++            v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gpu, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
+         }
+ 
+         map_layer_ids[il] = layers.size();
+ 
+-        layers.push_back({ il, k, v, k_stream, v_stream, });
++        layers.push_back({ il, k, v, k_stream, v_stream, k_cpu, v_cpu, gpu_heads });
+     }
+ 
+     if (reuse) {
+@@ -440,6 +485,20 @@ void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) {
+                 const size_t sz  = (size_t) (b - a) * (size_t) layer.v->nb[1];
+                 ggml_backend_tensor_memset(layer.v, 0, off, sz);
+             }
++            // opencoti F5 M2 headinfer — clear the host-resident head subset
++            // too (its own strides; same [a, b) cell range).
++            if (layer.k_cpu) {
++                const size_t off = (size_t) st * (size_t) layer.k_cpu->nb[2]
++                                 + (size_t) a  * (size_t) layer.k_cpu->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) layer.k_cpu->nb[1];
++                ggml_backend_tensor_memset(layer.k_cpu, 0, off, sz);
++            }
++            if (layer.v_cpu) {
++                const size_t off = (size_t) st * (size_t) layer.v_cpu->nb[2]
++                                 + (size_t) a  * (size_t) layer.v_cpu->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) layer.v_cpu->nb[1];
++                ggml_backend_tensor_memset(layer.v_cpu, 0, off, sz);
++            }
+         }
+     }
+ 
+@@ -610,6 +669,11 @@ void llama_kv_cache::shrink_if_idle() {
+     for (const auto & layer : layers) {
+         opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells);
+         opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells);
++        // opencoti F5 M2 headinfer — the host-resident head subset IS
++        // host-pageable, so this is where the GPU-mode shrink actually returns
++        // RSS (the device k/v above are skipped by the is_host() guard).
++        opencoti_decommit_layer_range(layer.k_cpu, n_stream, from_cells, to_cells);
++        opencoti_decommit_layer_range(layer.v_cpu, n_stream, from_cells, to_cells);
+     }
+ 
+     // After decommit, treat cells [from_cells, to_cells) as "uncleared" —
+@@ -1645,15 +1709,41 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
+ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+-    auto * k = layers[ikv].k;
++    const auto & layer = layers[ikv];
++    auto * k = layer.k;
++
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++
++    // opencoti F5 M2 headinfer — reassemble the full head set from the
++    // GPU-resident (k) and host-resident (k_cpu) subsets by concat on the head
++    // dimension. The backend scheduler streams k_cpu to the compute backend.
++    if (layer.gpu_heads > 0 && layer.k_cpu) {
++        const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++        const uint32_t n_head_gpu    = layer.gpu_heads;
++        const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
++        const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_k;
++        const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_k;
++        auto * kc = layer.k_cpu;
++        ggml_tensor * kg_v = ggml_view_4d(ctx, k,
++                n_embd_head_k, n_head_gpu, n_kv, ns,
++                ggml_row_size(k->type, n_embd_head_k),
++                ggml_row_size(k->type, n_embd_gpu),
++                ggml_row_size(k->type, n_embd_gpu*kv_size),
++                ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0);
++        ggml_tensor * kc_v = ggml_view_4d(ctx, kc,
++                n_embd_head_k, n_head_cpu, n_kv, ns,
++                ggml_row_size(kc->type, n_embd_head_k),
++                ggml_row_size(kc->type, n_embd_cpu),
++                ggml_row_size(kc->type, n_embd_cpu*kv_size),
++                ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0);
++        return ggml_concat(ctx, kg_v, kc_v, 1);
++    }
+ 
+-    const uint64_t kv_size      = get_size();
+     const uint64_t n_embd_k_gqa = k->ne[0];
+ 
+     assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il));
+ 
+-    const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
+-
+     return ggml_view_4d(ctx, k,
+             hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns,
+             ggml_row_size(k->type, hparams.n_embd_head_k(il)),
+@@ -1665,16 +1755,42 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k
+ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+-    auto * v = layers[ikv].v;
++    const auto & layer = layers[ikv];
++    auto * v = layer.v;
++
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++
++    // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The split
++    // is gated off when v_trans at construction, so this branch is always the
++    // non-transposed layout (heads in dim 1, same as get_k).
++    if (layer.gpu_heads > 0 && layer.v_cpu) {
++        const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++        const uint32_t n_head_gpu    = layer.gpu_heads;
++        const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
++        const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_v;
++        const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_v;
++        auto * vc = layer.v_cpu;
++        ggml_tensor * vg_v = ggml_view_4d(ctx, v,
++                n_embd_head_v, n_head_gpu, n_kv, ns,
++                ggml_row_size(v->type, n_embd_head_v),
++                ggml_row_size(v->type, n_embd_gpu),
++                ggml_row_size(v->type, n_embd_gpu*kv_size),
++                ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0);
++        ggml_tensor * vc_v = ggml_view_4d(ctx, vc,
++                n_embd_head_v, n_head_cpu, n_kv, ns,
++                ggml_row_size(vc->type, n_embd_head_v),
++                ggml_row_size(vc->type, n_embd_cpu),
++                ggml_row_size(vc->type, n_embd_cpu*kv_size),
++                ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0);
++        return ggml_concat(ctx, vg_v, vc_v, 1);
++    }
+ 
+-    const uint64_t kv_size      = get_size();
+     const uint64_t n_embd_v_gqa = v->ne[0];
+ 
+     // [TAG_V_CACHE_VARIABLE]
+     assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il));
+ 
+-    const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
+-
+     if (!v_trans) {
+         // note: v->nb[1] <= v->nb[2]
+         return ggml_view_4d(ctx, v,
+@@ -1699,7 +1815,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
+ 
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+-    ggml_tensor * k = layers[ikv].k;
++    const auto & layer = layers[ikv];
++    ggml_tensor * k = layer.k;
+ 
+     const int64_t n_embd_head = k_cur->ne[0];
+     const int64_t n_head      = k_cur->ne[1];
+@@ -1711,6 +1828,24 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
+     // TODO: add ggml helper function for this?
+     GGML_ASSERT(ggml_row_size(k_cur->type, n_embd_head) == k_cur->nb[1]);
+ 
++    // opencoti F5 M2 headinfer — scatter the incoming heads into the GPU and
++    // host subsets. The first gpu_heads heads (contiguous in k_cur's dim 0..1)
++    // store into k; the rest store into k_cpu, both at the same cell indices
++    // (k_idxs). The single returned dep node ties both set_rows into the
++    // caller's one ggml_build_forward_expand (its value is never read). Split
++    // only happens for n_stream==1 (gated at construction) — no per-stream
++    // reshape needed.
++    if (layer.gpu_heads > 0 && layer.k_cpu) {
++        const int64_t g = layer.gpu_heads;
++        const int64_t c = n_head - g;
++        ggml_tensor * kc = layer.k_cpu;
++        ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur, n_embd_head*g, n_tokens, k_cur->nb[2], 0);
++        ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur, n_embd_head*c, n_tokens, k_cur->nb[2], (size_t) (g*k_cur->nb[1]));
++        ggml_tensor * store_g = ggml_set_rows(ctx, k,  cur_g, k_idxs);
++        ggml_tensor * store_c = ggml_set_rows(ctx, kc, cur_c, k_idxs);
++        return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0));
++    }
++
+     k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0);
+ 
+     const int64_t n_stream = k->ne[2];
+@@ -1734,7 +1869,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
+ 
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+-    auto * v = layers[ikv].v;
++    const auto & layer = layers[ikv];
++    auto * v = layer.v;
+ 
+     const int64_t n_embd_head = v_cur->ne[0];
+     const int64_t n_head      = v_cur->ne[1];
+@@ -1745,6 +1881,20 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
+     // we can merge dims 0 and 1
+     GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]);
+ 
++    // opencoti F5 M2 headinfer — head-scatter into the GPU + host subsets.
++    // Reached only when !v_trans and n_stream==1 (the split is gated off
++    // otherwise at construction), so this mirrors the cpy_k store exactly.
++    if (layer.gpu_heads > 0 && layer.v_cpu) {
++        const int64_t g = layer.gpu_heads;
++        const int64_t c = n_head - g;
++        ggml_tensor * vc = layer.v_cpu;
++        ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur, n_embd_head*g, n_tokens, v_cur->nb[2], 0);
++        ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur, n_embd_head*c, n_tokens, v_cur->nb[2], (size_t) (g*v_cur->nb[1]));
++        ggml_tensor * store_g = ggml_set_rows(ctx, v,  cur_g, v_idxs);
++        ggml_tensor * store_c = ggml_set_rows(ctx, vc, cur_c, v_idxs);
++        return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0));
++    }
++
+     const int64_t n_stream = v->ne[2];
+ 
+     // take this branch when FA is enabled (the V cache is not transposed)
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -111,6 +111,11 @@ public:
+                      // back to kv_size_initial and the over-cap pages are
+                      // decommitted. 0 disables shrink-on-idle.
+                      uint32_t   slot_shrink_idle_ms,
++                     // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++                     // Fraction of KV heads kept GPU-resident (rest offloaded
++                     // to host, streamed back per step). 1.0 = off. Effective
++                     // only for offloaded layers with !v_trans (flash-attn).
++                     float      headinfer_gpu_heads_frac,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_pad,
+                      uint32_t   n_swa,
+@@ -269,6 +274,14 @@ private:
+ 
+         std::vector<ggml_tensor *> k_stream;
+         std::vector<ggml_tensor *> v_stream;
++
++        // opencoti F5 M2 headinfer — head-residency split. When gpu_heads > 0,
++        // `k`/`v` hold the first gpu_heads KV heads on the layer's (GPU) buffer
++        // and `k_cpu`/`v_cpu` hold the remaining (n_head_kv - gpu_heads) heads
++        // on a host buffer. gpu_heads == 0 means no split (k/v hold all heads).
++        ggml_tensor * k_cpu = nullptr;
++        ggml_tensor * v_cpu = nullptr;
++        uint32_t gpu_heads = 0;
+     };
+ 
+     bool v_trans = true;  // the value tensor is transposed
+diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp
++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+@@ -48,6 +48,8 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
+         // Hybrid-iswa does not yet thread slot_shrink_idle_ms; pass 0 for
+         // Phase 2 backward-compat (no shrink-on-idle).
+         0,
++        // opencoti F5 M2 headinfer — hybrid-iswa does not thread headinfer; 1.0 = off.
++        1.0f,
+         n_seq_max,
+         n_ubatch,
+         n_pad,
+diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp
+--- a/llama.cpp/src/llama-memory-hybrid.cpp
++++ b/llama.cpp/src/llama-memory-hybrid.cpp
+@@ -47,6 +47,8 @@ llama_memory_hybrid::llama_memory_hybrid(
+         // Hybrid memory does not yet thread slot_shrink_idle_ms; pass 0 for
+         // Phase 2 backward-compat (no shrink-on-idle).
+         0,
++        // opencoti F5 M2 headinfer — hybrid memory does not thread headinfer; 1.0 = off.
++        1.0f,
+         n_seq_max,
+         n_pad,
+         n_swa,
+diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp
+--- a/llama.cpp/src/llama-model.cpp
++++ b/llama.cpp/src/llama-model.cpp
+@@ -2077,6 +2077,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 cparams.slot_initial_ctx,
+                                 // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+                                 cparams.slot_shrink_idle_ms,
++                                // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++                                cparams.headinfer_gpu_heads_frac,
+                                 cparams.n_seq_max,
+                                 cparams.n_ubatch,
+                                 1,
+@@ -2097,6 +2099,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
+                                 cparams.slot_initial_ctx,
+                                 // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+                                 cparams.slot_shrink_idle_ms,
++                                // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++                                cparams.headinfer_gpu_heads_frac,
+                                 cparams.n_seq_max,
+                                 1,
+                                 hparams.n_swa,

patches/0030-neo-pipeline.patch

@@ -0,0 +1,933 @@
+From: opencoti
+Subject: [PATCH 0030] F5 M3 — NEO: asymmetric GPU/CPU attention pipelining
+
+Sits on top of 0020 (M2 head-residency split). When --neo-pipeline != off and
+the headinfer split is active for the layer, build_attn_mha builds TWO
+ggml_flash_attn_ext ops on disjoint Q-head ranges instead of M2's
+concat-then-stream-back path: cur_g consumes the GPU half of K/V on the GPU
+backend; cur_c consumes the CPU half of K/V on the CPU backend (auto-routed
+by the scheduler from each FA op's K/V buffer-type). Output halves are
+ggml_concat'd on the head axis — same head-axis concat M2 used, but on
+O(n_tokens) FA outputs instead of O(n_kv) K/V.
+
+A small orchestrator (ggml/include/ggml-neo-pipeline.h + ggml/src/
+ggml-neo-pipeline.cpp) lets the graph builder register each (cur_g, cur_c)
+pair; the CUDA backend's graph_compute hook routes registered cur_g to a
+non-default CUDA stream (curr_stream_no=1) so concurrent CPU FA on cur_c can
+overlap with stream-1 GPU FA. A cudaEvent is recorded on stream 1 after the
+GPU FA dispatch; when a downstream split's graph reads cur_g (typically the
+concat join), the hook inserts a cudaStreamWaitEvent on the consuming stream
+so the read sees completed data. CUDA graph capture is disabled for
+NEO-engaged splits; off-path graphs keep the fast cuda_graph path.
+
+Default off in every layer: cparams.neo_pipeline_mode == 0 means
+build_attn_mha_neo is never called and the M3-A structural split never
+engages — the M2 concat-then-stream-back path runs unchanged, byte-identical
+to the 0020-shipped binary. Adapter exposes --neo-pipeline off|on|auto via
+common/arg.cpp; @opencoti/llamafile typed config field
+neoPipelineMode + OPENCOTI_LLAMAFILE_NEO_PIPELINE env mirror the M2
+template.
+
+Solidpc RTX 3090 + Qwen2.5-Coder-0.5B IQ4_XS bench (perf/llamafile/
+neo-pipeline.bench.ts) at ctx=4096 / predict=128 / frac=0.5:
+
+  B baseline (no split)        326.69 tps
+  M m2-only  (split, M2 path)  186.96 tps
+  N neo-on   (split, M3 path)  187.04 tps  (+0.04% vs M)  ← C2 FAIL
+  O off-id   (--neo-pipeline off) 151.65 tps
+
+  C1 PASS  N completion === B completion (correctness intact)
+  C2 FAIL  N tps - M tps < 5% (NEO overlap mechanism is correct but the
+           per-step GPU FA on half-heads finishes in much less wall time
+           than the CPU FA on half-heads, so even perfect overlap is
+           bounded by the slower CPU half — for this model + ctx,
+           M3-B's stream-swap has no observable win over M3-A's
+           structural split alone)
+  C3 PASS  N GPU KV (24 MiB) ≈ M GPU KV (24 MiB) — residency preserved
+  C4 PASS  O completion === M completion (--neo-pipeline off byte-identical)
+
+Per the M3-D gate in docs/features/advanced_kv.md (F5 M3): C2 failed but
+the orchestrator is correct and the off-path is byte-identical to M2,
+so the orchestrator file ships as a wired-but-no-observable-win
+mechanism (kept for M4 ScoutAttention reuse) rather than as a
+dormant stub. The M3-A structural win (per-step CPU→GPU stream-back
+eliminated) is the actual deliverable. M3-C EWMA load-aware controller
+is deferred — without an observable M3-B overlap win it would be
+cosmetic.
+
+Surgical-hook count stays 18 — all 0030 changes are in vendored-source
+files, captured here via snapshot-diff (NOT git diff HEAD; bug-121).
+
+diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
+--- a/llama.cpp/BUILD.mk
++++ b/llama.cpp/BUILD.mk
+@@ -27,6 +27,7 @@ GGML_SRCS_CPP := \
+ 	llama.cpp/ggml/src/ggml-backend-meta.cpp \
+ 	llama.cpp/ggml/src/ggml-backend-reg.cpp \
+ 	llama.cpp/ggml/src/ggml-backend.cpp \
++	llama.cpp/ggml/src/ggml-neo-pipeline.cpp \
+ 	llama.cpp/ggml/src/ggml-opt.cpp \
+ 	llama.cpp/ggml/src/ggml-threading.cpp \
+ 	llama.cpp/ggml/src/ggml.cpp \
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1429,6 +1429,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.headinfer_gpu_heads_frac = std::stof(value);
+         }
+     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC"));
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--neo-pipeline"}, "MODE",
++        "neo (asymmetric GPU/CPU attention pipelining): off (default), on (engage when headinfer split is active for the layer), auto (reserved, same as on). Requires --headinfer-gpu-heads-frac < 1.0 to do anything.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "off")  params.neo_pipeline_mode = 0;
++            else if (value == "on")   params.neo_pipeline_mode = 1;
++            else if (value == "auto") params.neo_pipeline_mode = 2;
++            else throw std::invalid_argument("--neo-pipeline must be off|on|auto");
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE"));
+     add_opt(common_arg(
+         {"--chunks"}, "N",
+         string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
+diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp
+--- a/llama.cpp/common/common.cpp
++++ b/llama.cpp/common/common.cpp
+@@ -1634,6 +1634,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
+     cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms;
+     // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
+     cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    cparams.neo_pipeline_mode = params.neo_pipeline_mode;
+     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+     cparams.n_batch           = params.n_batch;
+     cparams.n_ubatch          = params.n_ubatch;
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -440,6 +440,11 @@ struct common_params {
+     // memory and streamed back per step. 1.0 = off (no split). Only active for
+     // GPU-offloaded layers with flash-attention (non-transposed V cache).
+     float   headinfer_gpu_heads_frac = 1.0f;
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    // 0 = off (default; M2 concat path runs), 1 = on (engage when split
++    // active for the layer), 2 = auto (reserved for future load-based
++    // engagement; same as `on` today).
++    uint32_t neo_pipeline_mode    =     0;
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
+diff --git a/llama.cpp/ggml/include/ggml-neo-pipeline.h b/llama.cpp/ggml/include/ggml-neo-pipeline.h
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/include/ggml-neo-pipeline.h
+@@ -0,0 +1,78 @@
++// opencoti F5 M3 NEO — asymmetric GPU/CPU attention pipelining.
++//
++// This header declares the orchestrator surface that lets the M3 two-flash-attn
++// structural split overlap its GPU and CPU halves. The graph builder registers
++// each (cur_g, cur_c) pair after constructing it; the CUDA backend consults the
++// registry at compute time to (a) dispatch GPU FA on a non-default CUDA stream
++// so the scheduler's stream-0 sync no longer blocks the CPU FA's input copy
++// and (b) wait on the recorded stream-1 event when a downstream split consumes
++// cur_g (notably the concat that joins both halves), keeping correctness intact.
++//
++// Default-off: ggml_neo_pipeline_set_enabled(false) makes register_pair a no-op
++// and the CUDA hooks fast-skip. The off-path is byte-identical to the M3-A
++// shipped path; M3-A in turn is byte-identical to M2's concat path when
++// --neo-pipeline off (cparams.neo_pipeline_mode == 0). See
++// docs/features/advanced_kv.md (F5 M3 NEO).
++#pragma once
++
++#include "ggml.h"
++
++#ifdef __cplusplus
++extern "C" {
++#endif
++
++// Global on/off switch. Off by default; the graph builder flips this on for
++// the duration of a graph build when cparams.neo_pipeline_mode != 0 and at
++// least one layer has an active headinfer split. While off, all functions in
++// this file are O(1) no-ops.
++GGML_API void ggml_neo_pipeline_set_enabled(bool enabled);
++GGML_API bool ggml_neo_pipeline_enabled(void);
++
++// Registers a NEO FA pair. Called from build_attn_mha_neo after constructing
++// the two flash-attention ops on disjoint head ranges. cur_g lives on the GPU
++// backend (CUDA-allocated K/V), cur_c on the CPU backend (CPU-allocated K/V).
++// The CUDA backend later consults this registry to route cur_g to a non-default
++// stream so cur_c can dispatch concurrently. When disabled, this is a no-op.
++GGML_API void ggml_neo_pipeline_register_pair(const struct ggml_tensor * cur_g,
++                                              const struct ggml_tensor * cur_c);
++
++// Membership probe. Returns true iff t was the cur_g argument of a prior
++// register_pair call in this graph. O(1) fast-path; the CUDA backend calls
++// this once per node when scanning a graph it is about to dispatch.
++GGML_API bool ggml_neo_pipeline_is_gpu_partner(const struct ggml_tensor * t);
++
++// CUDA event slot for cur_g. The CUDA backend creates a cudaEvent (with
++// cudaEventCreateWithFlags + cudaEventDisableTiming) the first time it
++// dispatches cur_g on a non-default stream and stores the opaque handle via
++// set_event. A later split whose graph reads cur_g calls get_event and
++// inserts a cudaStreamWaitEvent so the consuming stream waits for the
++// stream-1 work. Both functions are no-ops when NEO is disabled or t isn't
++// registered (get_event returns NULL). The void* is treated as cudaEvent_t
++// at the CUDA call site; the orchestrator does not know the underlying type.
++GGML_API void * ggml_neo_pipeline_get_event(const struct ggml_tensor * cur_g);
++GGML_API void   ggml_neo_pipeline_set_event(const struct ggml_tensor * cur_g, void * event);
++
++// Clears all per-graph registrations. Called by the CUDA backend at the end
++// of each graph dispatch so registrations don't leak across decode steps.
++// (The graph builder reinstalls them on the next build.) Notably does NOT
++// destroy stored events — those are destroyed by the CUDA backend, which
++// owns the cudaEvent lifecycle (call destroy_all_events at shutdown).
++GGML_API void ggml_neo_pipeline_clear_registrations(void);
++
++// Walks every (cur_g, event) entry, invoking destroyer(event) for each
++// non-null event, then clears the event map. Called at session/backend
++// shutdown so cudaEvents are properly released. destroyer is typically a
++// thin lambda wrapping cudaEventDestroy at the CUDA call site.
++GGML_API void ggml_neo_pipeline_destroy_all_events(void (*destroyer)(void * event));
++
++// Counter of NEO pair dispatches observed by the CUDA backend in the current
++// process. Surfaced for the M3-E bench's residency / engagement checks.
++GGML_API int  ggml_neo_pipeline_dispatch_count(void);
++
++// Increments the dispatch counter. Called by the CUDA backend each time it
++// observes and routes a registered cur_g to a non-default stream.
++GGML_API void ggml_neo_pipeline_inc_dispatch_count(void);
++
++#ifdef __cplusplus
++}
++#endif
+diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+@@ -2,6 +2,10 @@
+ #include "ggml-impl.h"
+ #include "ggml-backend-impl.h"
+ 
++// opencoti F5 M3-B NEO orchestrator (registry of cur_g/cur_c pairs for the
++// asymmetric attention pipelining; consulted in graph_compute below).
++#include "ggml-neo-pipeline.h"
++
+ #include "ggml-cuda/allreduce.cuh"
+ #include "ggml-cuda/common.cuh"
+ #include "ggml-cuda/acc.cuh"
+@@ -4485,6 +4489,58 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t
+ 
+     ggml_cuda_set_device(cuda_ctx->device);
+ 
++    // opencoti F5 M3-B NEO: scan the graph for registered cur_g flash-attn
++    // outputs and for downstream consumers of prior cur_g. The orchestrator
++    // is dormant unless the graph builder called set_enabled(true) (which
++    // only happens when --neo-pipeline != off AND at least one layer has
++    // an active headinfer split). When dormant this loop body is one
++    // atomic-load fast-skip.
++    //
++    // When NEO is engaged for this split:
++    //   - If the split computes a cur_g, swap curr_stream_no to 1 so the
++    //     kernels dispatch on the alternate CUDA stream; the scheduler's
++    //     stream-0 sync in compute_splits will NOT block on this work, so
++    //     the subsequent split's q_c host copy starts immediately and the
++    //     CPU FA on cur_c can overlap with our stream-1 work. We record an
++    //     event on stream 1 at exit so downstream consumers know when
++    //     cur_g is done.
++    //   - If the split reads a prior cur_g (typically the concat join),
++    //     stream-wait on the recorded event before kernels read it.
++    //
++    // CUDA graph capture is disabled when NEO is active for this split:
++    // capturing kernel dispatch on a non-default stream interacts with the
++    // graph_key cache and the existing concurrent_event re-ordering pass
++    // in ways we haven't validated. Off-path graphs keep the cuda_graph
++    // fast path; only NEO-engaged splits pay this cost.
++    bool          neo_active           = false;
++    ggml_tensor * neo_cur_g_to_record  = nullptr;
++    if (ggml_neo_pipeline_enabled()) {
++        for (int i = 0; i < cgraph->n_nodes; ++i) {
++            ggml_tensor * node = cgraph->nodes[i];
++            if (ggml_neo_pipeline_is_gpu_partner(node)) {
++                neo_active          = true;
++                neo_cur_g_to_record = node;
++                break;
++            }
++            for (int s = 0; s < GGML_MAX_SRC; ++s) {
++                if (node->src[s] != nullptr && ggml_neo_pipeline_is_gpu_partner(node->src[s])) {
++                    void * ev = ggml_neo_pipeline_get_event(node->src[s]);
++                    if (ev != nullptr) {
++                        CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t) ev, 0));
++                    }
++                }
++            }
++        }
++    }
++    if (neo_active) {
++        // Materialize stream 1 (creates it on first use) and switch the
++        // context to dispatch all subsequent kernels there. Both calls are
++        // cheap: stream() is a vector lookup + lazy create; the curr_stream_no
++        // write is a single int.
++        (void) cuda_ctx->stream(cuda_ctx->device, 1);
++        cuda_ctx->curr_stream_no = 1;
++    }
++
+     bool use_cuda_graph             = false;
+     bool cuda_graph_update_required = false;
+     const void * graph_key = nullptr;
+@@ -4524,6 +4580,16 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t
+     }
+ #endif // USE_CUDA_GRAPH
+ 
++    // opencoti F5 M3-B NEO: force-disable cuda_graph capture for NEO-active
++    // splits. Capturing on a non-default stream is structurally fine in CUDA
++    // but interacts with the existing graph_key cache + concurrent_event
++    // re-ordering pass in ways we haven't validated. Off-path graphs keep
++    // the fast cuda_graph path.
++    if (neo_active) {
++        use_cuda_graph             = false;
++        cuda_graph_update_required = false;
++    }
++
+     if (use_cuda_graph && cuda_graph_update_required) {
+         // Start CUDA graph capture
+         {
+@@ -4536,6 +4602,23 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t
+ 
+     ggml_cuda_graph_evaluate_and_capture(cuda_ctx, cgraph, use_cuda_graph, cuda_graph_update_required, graph_key);
+ 
++    // opencoti F5 M3-B NEO exit: record an event on stream 1 marking cur_g's
++    // completion so the downstream concat (consumer split) can stream-wait
++    // on it from stream 0. Restore curr_stream_no=0 so the next graph_compute
++    // call starts on the default stream as normal.
++    if (neo_active && neo_cur_g_to_record != nullptr) {
++        void * ev = ggml_neo_pipeline_get_event(neo_cur_g_to_record);
++        if (ev == nullptr) {
++            cudaEvent_t new_ev;
++            CUDA_CHECK(cudaEventCreateWithFlags(&new_ev, cudaEventDisableTiming));
++            ev = (void *) new_ev;
++            ggml_neo_pipeline_set_event(neo_cur_g_to_record, ev);
++        }
++        CUDA_CHECK(cudaEventRecord((cudaEvent_t) ev, cuda_ctx->stream(cuda_ctx->device, 1)));
++        ggml_neo_pipeline_inc_dispatch_count();
++        cuda_ctx->curr_stream_no = 0;
++    }
++
+     return GGML_STATUS_SUCCESS;
+ }
+ 
+diff --git a/llama.cpp/ggml/src/ggml-neo-pipeline.cpp b/llama.cpp/ggml/src/ggml-neo-pipeline.cpp
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-neo-pipeline.cpp
+@@ -0,0 +1,126 @@
++// opencoti F5 M3 NEO orchestrator implementation.
++//
++// Backend-agnostic registry that lets the CUDA backend recognize the M3-built
++// (cur_g, cur_c) flash-attention pairs and route cur_g to a non-default CUDA
++// stream so cur_c can dispatch on the CPU backend concurrently.
++//
++// The orchestrator stores tensor pointers and opaque cudaEvent handles. It
++// does not call any CUDA API directly — the CUDA backend creates events with
++// cudaEventCreateWithFlags and hands them to set_event; on shutdown the CUDA
++// backend supplies a destroyer callback that wraps cudaEventDestroy.
++//
++// All maps are protected by a single mutex. Lookups are O(1) (unordered_map).
++// While the global enabled flag is false, every public function fast-skips
++// without touching the maps, so the off-path is one atomic-load + branch.
++//
++// See ggml/include/ggml-neo-pipeline.h for the public API contract and
++// docs/features/advanced_kv.md (F5 M3) for the surrounding design.
++
++#include "ggml-neo-pipeline.h"
++
++#include <atomic>
++#include <mutex>
++#include <unordered_map>
++#include <unordered_set>
++
++namespace {
++
++std::atomic<bool> g_enabled{false};
++std::atomic<int>  g_dispatch_count{0};
++
++std::mutex g_mu;
++
++// Registered cur_g tensors. Membership probe is O(1).
++std::unordered_set<const ggml_tensor *> g_gpu_partners;
++
++// Map from cur_g → cudaEvent_t handle (opaque void *). NULL slot means the
++// CUDA backend hasn't created an event for this tensor yet — get_event
++// returns nullptr in that case so the caller knows to create-and-set.
++std::unordered_map<const ggml_tensor *, void *> g_events;
++
++} // namespace
++
++void ggml_neo_pipeline_set_enabled(bool enabled) {
++    g_enabled.store(enabled, std::memory_order_release);
++}
++
++bool ggml_neo_pipeline_enabled(void) {
++    return g_enabled.load(std::memory_order_acquire);
++}
++
++void ggml_neo_pipeline_register_pair(const ggml_tensor * cur_g, const ggml_tensor * cur_c) {
++    (void) cur_c; // cpu partner is informational; the orchestrator only acts on cur_g
++    if (!ggml_neo_pipeline_enabled()) {
++        return;
++    }
++    if (cur_g == nullptr) {
++        return;
++    }
++    std::lock_guard<std::mutex> lock(g_mu);
++    g_gpu_partners.insert(cur_g);
++}
++
++bool ggml_neo_pipeline_is_gpu_partner(const ggml_tensor * t) {
++    if (!ggml_neo_pipeline_enabled()) {
++        return false;
++    }
++    if (t == nullptr) {
++        return false;
++    }
++    std::lock_guard<std::mutex> lock(g_mu);
++    return g_gpu_partners.find(t) != g_gpu_partners.end();
++}
++
++void * ggml_neo_pipeline_get_event(const ggml_tensor * cur_g) {
++    if (!ggml_neo_pipeline_enabled()) {
++        return nullptr;
++    }
++    if (cur_g == nullptr) {
++        return nullptr;
++    }
++    std::lock_guard<std::mutex> lock(g_mu);
++    const auto it = g_events.find(cur_g);
++    if (it == g_events.end()) {
++        return nullptr;
++    }
++    return it->second;
++}
++
++void ggml_neo_pipeline_set_event(const ggml_tensor * cur_g, void * event) {
++    if (!ggml_neo_pipeline_enabled()) {
++        return;
++    }
++    if (cur_g == nullptr) {
++        return;
++    }
++    std::lock_guard<std::mutex> lock(g_mu);
++    g_events[cur_g] = event;
++}
++
++void ggml_neo_pipeline_clear_registrations(void) {
++    std::lock_guard<std::mutex> lock(g_mu);
++    g_gpu_partners.clear();
++    // Note: g_events deliberately NOT cleared here. Events are reusable
++    // across decode steps as long as the underlying tensor pointers stay
++    // stable. They're cleaned up by destroy_all_events on shutdown.
++}
++
++void ggml_neo_pipeline_destroy_all_events(void (*destroyer)(void * event)) {
++    std::lock_guard<std::mutex> lock(g_mu);
++    if (destroyer != nullptr) {
++        for (auto & kv : g_events) {
++            if (kv.second != nullptr) {
++                destroyer(kv.second);
++            }
++        }
++    }
++    g_events.clear();
++}
++
++int ggml_neo_pipeline_dispatch_count(void) {
++    return g_dispatch_count.load(std::memory_order_acquire);
++}
++
++void ggml_neo_pipeline_inc_dispatch_count(void) {
++    g_dispatch_count.fetch_add(1, std::memory_order_relaxed);
++}
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -358,6 +358,10 @@ extern "C" {
+         // streamed back per step). 1.0 = off. Only effective for offloaded
+         // layers with flash-attention (non-transposed V cache).
+         float    headinfer_gpu_heads_frac;
++        // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++        // 0 = off (default), 1 = on, 2 = auto. Effective only when the
++        // M2 head split is active for a given layer.
++        uint32_t neo_pipeline_mode;
+         uint32_t n_rs_seq;          // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
+         int32_t  n_threads;         // number of threads to use for generation
+         int32_t  n_threads_batch;   // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -64,6 +64,8 @@ llama_context::llama_context(
+     cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms;
+     // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
+     cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    cparams.neo_pipeline_mode = params.neo_pipeline_mode;
+ 
+     cparams.n_threads        = params.n_threads;
+     cparams.n_threads_batch  = params.n_threads_batch;
+@@ -3356,6 +3358,7 @@ llama_context_params llama_context_default_params() {
+         /*.slot_initial_ctx            =*/ 0,
+         /*.slot_shrink_idle_ms         =*/ 0,
+         /*.headinfer_gpu_heads_frac    =*/ 1.0f,
++        /*.neo_pipeline_mode            =*/ 0,  // opencoti F5 M3 neo — off by default
+         /*.n_rs_seq                    =*/ 0,
+         /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+         /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -23,6 +23,15 @@ struct llama_cparams {
+     uint32_t slot_shrink_idle_ms;
+     // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
+     float    headinfer_gpu_heads_frac;
++    // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention.
++    // 0 = off (M2 concat path runs; binary equivalent to upstream when
++    //         headinfer_gpu_heads_frac == 1.0).
++    // 1 = on  (engage when headinfer split is active for the layer:
++    //         build two flash_attn_ext ops on disjoint head ranges, run
++    //         them on GPU and CPU backends respectively, concat outputs).
++    // 2 = auto (same engagement as `on` for now — reserved for future
++    //         load-based engagement heuristics).
++    uint32_t neo_pipeline_mode;
+     uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+     int32_t  n_threads;       // number of threads to use for generation
+     int32_t  n_threads_batch; // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp
+--- a/llama.cpp/src/llama-graph.cpp
++++ b/llama.cpp/src/llama-graph.cpp
+@@ -11,6 +11,10 @@
+ #include "llama-memory-hybrid-iswa.h"
+ #include "llama-memory-recurrent.h"
+ 
++// opencoti F5 M3 NEO orchestrator (registers cur_g/cur_c pairs so the CUDA
++// backend can route GPU FA to a non-default stream and overlap with CPU FA).
++#include "ggml-neo-pipeline.h"
++
+ #include <cassert>
+ #include <cmath>
+ #include <cstring>
+@@ -2085,6 +2089,140 @@ ggml_tensor * llm_graph_context::build_attn_mha(
+     return cur;
+ }
+ 
++// opencoti F5 M3 neo — head-axis-split attention. Builds two flash_attn_ext
++// ops on disjoint head ranges:
++//   GPU side: q[..., :gpu_heads*gqa, ...] × k_g × v_g  (auto-routes to GPU
++//             via the scheduler because k_g/v_g are views on the layer's
++//             GPU buffer-type tensor — see ggml-backend.cpp:797-852).
++//   CPU side: q[..., gpu_heads*gqa:, ...] × k_c × v_c  (auto-routes to CPU
++//             because k_c/v_c are views on the CPU buffer-type tensor that
++//             M2's headinfer allocates alongside the GPU half — see
++//             llama-kv-cache.cpp:get_k_cpu/get_v_cpu).
++// Outputs are concatenated on the head axis and fed into the standard
++// reshape_2d → Wo flow. This eliminates M2's per-step CPU→GPU stream-back
++// (O(n_kv) bytes per layer) — only the small O(n_tokens) CPU FA output
++// is bounced to GPU for the concat. Pre-M3-B, the two FA ops run serially
++// under the scheduler's default dispatch; M3-B's orchestrator will overlap
++// them concurrently.
++ggml_tensor * llm_graph_context::build_attn_mha_neo(
++        ggml_tensor * q,
++        ggml_tensor * k_g,
++        ggml_tensor * v_g,
++        ggml_tensor * k_c,
++        ggml_tensor * v_c,
++            uint32_t  gpu_heads,
++        ggml_tensor * kq_b,
++        ggml_tensor * kq_mask,
++        ggml_tensor * sinks,
++        ggml_tensor * v_mla,
++              float   kq_scale,
++                int   il) const {
++    // Preconditions enforced by the M2 split gate that produced k_c/v_c.
++    GGML_ASSERT(cparams.flash_attn && "NEO path requires flash-attn");
++    GGML_ASSERT(kq_b == nullptr     && "NEO path does not support KQ bias");
++    GGML_ASSERT(v_mla == nullptr    && "NEO path does not support MLA");
++    GGML_ASSERT(gpu_heads > 0       && "NEO path requires an active head split");
++    GGML_ASSERT(k_g && v_g && k_c && v_c);
++
++    // K/V layout is identical between the GPU and CPU halves (non-transposed
++    // V per the M2 gate). Stream axis matches across both — both are views
++    // on tensors allocated with the same n_stream at construction.
++    GGML_ASSERT(k_g->ne[3] == k_c->ne[3]);
++    GGML_ASSERT(v_g->ne[3] == v_c->ne[3]);
++    GGML_ASSERT(k_g->nb[1] <= k_g->nb[2] && "headinfer split requires non-transposed V");
++
++    // The full Q has all n_head_q heads. Heads are split on KV-head-group
++    // boundaries (M2's GQA clamp ensures gpu_heads is a whole number of
++    // groups). gqa_ratio = n_head_q / n_head_kv; n_head_kv = gpu_heads +
++    // cpu_heads. Q-heads serving the same KV head are CONTIGUOUS in Q's
++    // dim 1 under standard GGUF, so the slice is a clean view (no extra
++    // permute beyond the one already in build_attn_mha).
++    const uint32_t n_head_q   = (uint32_t) q->ne[1];
++    const uint32_t n_head_kv  = (uint32_t) (k_g->ne[1] + k_c->ne[1]);
++    GGML_ASSERT(n_head_q % n_head_kv == 0 && "GQA ratio must divide n_head_q");
++    const uint32_t gqa_ratio  = n_head_q / n_head_kv;
++    const uint32_t gpu_q_heads = gpu_heads * gqa_ratio;
++    const uint32_t cpu_q_heads = (n_head_kv - gpu_heads) * gqa_ratio;
++    GGML_ASSERT(gpu_q_heads + cpu_q_heads == n_head_q);
++
++    // Mirror build_attn_mha's n_stream reshape + permute on Q, but slice Q
++    // on the head axis BEFORE the permute so the dim-1 slice stays
++    // contiguous in the input layout.
++    const auto n_stream = k_g->ne[3];
++    ggml_tensor * q_4 = ggml_view_4d(ctx0, q,
++            q->ne[0], q->ne[1], q->ne[2]/n_stream, n_stream,
++            q->nb[1], q->nb[2], q->nb[3]/n_stream, 0);
++
++    ggml_tensor * q_g = ggml_view_4d(ctx0, q_4,
++            q_4->ne[0], gpu_q_heads, q_4->ne[2], q_4->ne[3],
++            q_4->nb[1], q_4->nb[2], q_4->nb[3],
++            /*offset=*/ 0);
++    ggml_tensor * q_c = ggml_view_4d(ctx0, q_4,
++            q_4->ne[0], cpu_q_heads, q_4->ne[2], q_4->ne[3],
++            q_4->nb[1], q_4->nb[2], q_4->nb[3],
++            /*offset=*/ (size_t) gpu_q_heads * q_4->nb[1]);
++
++    q_g = ggml_permute(ctx0, q_g, 0, 2, 1, 3);
++    q_c = ggml_permute(ctx0, q_c, 0, 2, 1, 3);
++    ggml_tensor * k_g_p = ggml_permute(ctx0, k_g, 0, 2, 1, 3);
++    ggml_tensor * v_g_p = ggml_permute(ctx0, v_g, 0, 2, 1, 3);
++    ggml_tensor * k_c_p = ggml_permute(ctx0, k_c, 0, 2, 1, 3);
++    ggml_tensor * v_c_p = ggml_permute(ctx0, v_c, 0, 2, 1, 3);
++
++    // F32 K → F16 cast mirrors build_attn_mha lines 1881-1887 (the embedding
++    // model non-causal path). v_trans is asserted false above, so no extra
++    // transpose is needed; V already has the same dim-1 head layout as K.
++    if (k_g_p->type == GGML_TYPE_F32) k_g_p = ggml_cast(ctx0, k_g_p, GGML_TYPE_F16);
++    if (k_c_p->type == GGML_TYPE_F32) k_c_p = ggml_cast(ctx0, k_c_p, GGML_TYPE_F16);
++    if (v_g_p->type == GGML_TYPE_F32) v_g_p = ggml_cast(ctx0, v_g_p, GGML_TYPE_F16);
++    if (v_c_p->type == GGML_TYPE_F32) v_c_p = ggml_cast(ctx0, v_c_p, GGML_TYPE_F16);
++
++    // Sinks (if present) carry one F32 entry per Q-head; slice them to
++    // match each half. The flash_attn_ext_add_sinks op asserts
++    // sinks->ne[0] == q->src[0]->ne[2] (the q-head axis after permute).
++    ggml_tensor * sinks_g = nullptr;
++    ggml_tensor * sinks_c = nullptr;
++    if (sinks) {
++        GGML_ASSERT((uint32_t) sinks->ne[0] == n_head_q);
++        sinks_g = ggml_view_1d(ctx0, sinks, gpu_q_heads, 0);
++        sinks_c = ggml_view_1d(ctx0, sinks, cpu_q_heads,
++                               (size_t) gpu_q_heads * sinks->nb[0]);
++    }
++
++    const float alibi_bias  = hparams.f_max_alibi_bias;
++    const float logit_cap   = hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f;
++
++    ggml_tensor * cur_g = ggml_flash_attn_ext(ctx0, q_g, k_g_p, v_g_p, kq_mask, kq_scale, alibi_bias, logit_cap);
++    cb(cur_g, LLAMA_TENSOR_NAME_FATTN "_g", il);
++    ggml_flash_attn_ext_add_sinks(cur_g, sinks_g);
++    ggml_flash_attn_ext_set_prec (cur_g, GGML_PREC_F32);
++
++    ggml_tensor * cur_c = ggml_flash_attn_ext(ctx0, q_c, k_c_p, v_c_p, kq_mask, kq_scale, alibi_bias, logit_cap);
++    cb(cur_c, LLAMA_TENSOR_NAME_FATTN "_c", il);
++    ggml_flash_attn_ext_add_sinks(cur_c, sinks_c);
++    ggml_flash_attn_ext_set_prec (cur_c, GGML_PREC_F32);
++
++    // opencoti F5 M3-B: enable + register the pair so the CUDA backend swaps
++    // cur_g to a non-default stream at dispatch time, letting cur_c (on CPU)
++    // overlap. set_enabled(true) is idempotent and only flipped here — the
++    // call site already gates on neo_pipeline_mode != 0 && headinfer_split_active,
++    // so reaching this point means NEO is engaged for this layer.
++    ggml_neo_pipeline_set_enabled(true);
++    ggml_neo_pipeline_register_pair(cur_g, cur_c);
++
++    // FA output shape is [v_dim, n_q_heads_split, n_tokens, n_stream].
++    // Concat on dim 1 (q-head axis) to reassemble the full Q-head set.
++    // The scheduler synchronizes here — cur_c gets lifted to GPU so the
++    // concat runs on whichever backend hosts cur_g (typically GPU).
++    ggml_tensor * cur = ggml_concat(ctx0, cur_g, cur_c, 1);
++    cb(cur, LLAMA_TENSOR_NAME_FATTN, il);
++
++    // Match build_attn_mha's flash-attn final reshape (line 1913).
++    cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
++
++    return cur;
++}
++
+ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() const {
+     auto inp = std::make_unique<llm_graph_input_attn_no_cache>(hparams, cparams);
+ 
+@@ -2237,10 +2375,29 @@ ggml_tensor * llm_graph_context::build_attn(
+     const auto & kq_mask = inp->get_kq_mask();
+ 
+     ggml_tensor * q = q_cur;
+-    ggml_tensor * k = mctx_cur->get_k(ctx0, il);
+-    ggml_tensor * v = mctx_cur->get_v(ctx0, il);
+ 
+-    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
++    // opencoti F5 M3 neo — head-axis-split attention. When the M2 head
++    // residency split is active for this layer AND --neo-pipeline is on,
++    // build two flash_attn_ext ops on disjoint head ranges instead of the
++    // M2 concat-reassembly path (which streams k_cpu/v_cpu back to GPU each
++    // step and runs a single full-head FA op). The scheduler routes each
++    // FA op to the backend matching its K/V buffer-type. v_mla must be
++    // null (build_attn for KV asserts this above at :2102), kq_b too;
++    // the M2 gate ensures non-transposed V. See docs/features/advanced_kv.md.
++    ggml_tensor * cur;
++    if (cparams.neo_pipeline_mode != 0 && mctx_cur->headinfer_split_active(il)) {
++        ggml_tensor * k_g = mctx_cur->get_k_gpu(ctx0, il);
++        ggml_tensor * v_g = mctx_cur->get_v_gpu(ctx0, il);
++        ggml_tensor * k_c = mctx_cur->get_k_cpu(ctx0, il);
++        ggml_tensor * v_c = mctx_cur->get_v_cpu(ctx0, il);
++        cur = build_attn_mha_neo(q, k_g, v_g, k_c, v_c,
++                                 mctx_cur->headinfer_gpu_heads(il),
++                                 kq_b, kq_mask, sinks, v_mla, kq_scale, il);
++    } else {
++        ggml_tensor * k = mctx_cur->get_k(ctx0, il);
++        ggml_tensor * v = mctx_cur->get_v(ctx0, il);
++        cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
++    }
+     cb(cur, "kqv_out", il);
+ 
+     if (inp->self_v_rot) {
+diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h
+--- a/llama.cpp/src/llama-graph.h
++++ b/llama.cpp/src/llama-graph.h
+@@ -908,6 +908,27 @@ struct llm_graph_context {
+                   float   kq_scale,
+                     int   il) const;
+ 
++    // opencoti F5 M3 neo — head-axis split flash-attn. Builds two
++    // ggml_flash_attn_ext ops on disjoint head ranges (GPU vs CPU subsets,
++    // routed automatically by the scheduler via the K/V tensors' buffer
++    // types) and concatenates their outputs on the head axis before the
++    // reshape/Wo projection. Flash-attention only (assert kq_b == nullptr,
++    // v_mla == nullptr, !v_trans for V — the M2 split gate already enforces
++    // this). Output shape matches build_attn_mha's flash-attn branch.
++    ggml_tensor * build_attn_mha_neo(
++            ggml_tensor * q,       // [n_embd_head_q, n_head_q, n_tokens]
++            ggml_tensor * k_g,     // GPU-resident K subset (gpu_heads on dim 1)
++            ggml_tensor * v_g,     // GPU-resident V subset (gpu_heads on dim 1)
++            ggml_tensor * k_c,     // CPU-resident K subset (cpu_heads on dim 1)
++            ggml_tensor * v_c,     // CPU-resident V subset (cpu_heads on dim 1)
++                uint32_t  gpu_heads,
++            ggml_tensor * kq_b,    // must be nullptr
++            ggml_tensor * kq_mask,
++            ggml_tensor * sinks,   // [n_head_q] or nullptr
++            ggml_tensor * v_mla,   // must be nullptr
++                  float   kq_scale,
++                    int   il) const;
++
+     llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const;
+ 
+     ggml_tensor * build_attn(
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -1810,6 +1810,116 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k
+             ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0);
+ }
+ 
++// opencoti F5 M3 neo — head-split-aware accessors. These factor the per-half
++// view machinery that get_k/get_v perform internally before their concat,
++// exposing the GPU-only and CPU-only sub-tensors as independent ggml views.
++// M3 NEO uses them to build two flash_attn_ext ops (one per backend, scheduler
++// auto-routes by buffer type) and merge their outputs, eliminating M2's
++// per-step O(n_kv) CPU→GPU stream-back. Dimensions and strides are identical
++// to the corresponding halves inside M2's get_k/get_v.
++
++bool llama_kv_cache::headinfer_split_active(int32_t il) const {
++    const auto it = map_layer_ids.find(il);
++    if (it == map_layer_ids.end()) {
++        return false;
++    }
++    const auto & layer = layers[it->second];
++    return layer.gpu_heads > 0 && layer.k_cpu != nullptr;
++}
++
++uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const {
++    const auto it = map_layer_ids.find(il);
++    if (it == map_layer_ids.end()) {
++        return 0;
++    }
++    return layers[it->second].gpu_heads;
++}
++
++ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
++    const int32_t ikv = map_layer_ids.at(il);
++    const auto & layer = layers[ikv];
++    GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split");
++
++    auto * k = layer.k;
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++    const uint32_t n_head_gpu    = layer.gpu_heads;
++    const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_k;
++
++    return ggml_view_4d(ctx, k,
++            n_embd_head_k, n_head_gpu, n_kv, ns,
++            ggml_row_size(k->type, n_embd_head_k),
++            ggml_row_size(k->type, n_embd_gpu),
++            ggml_row_size(k->type, n_embd_gpu*kv_size),
++            ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0);
++}
++
++ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
++    const int32_t ikv = map_layer_ids.at(il);
++    const auto & layer = layers[ikv];
++    GGML_ASSERT(layer.gpu_heads > 0 && layer.k_cpu && "get_k_cpu called without an active headinfer split");
++
++    auto * kc = layer.k_cpu;
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++    const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
++    const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_k;
++
++    return ggml_view_4d(ctx, kc,
++            n_embd_head_k, n_head_cpu, n_kv, ns,
++            ggml_row_size(kc->type, n_embd_head_k),
++            ggml_row_size(kc->type, n_embd_cpu),
++            ggml_row_size(kc->type, n_embd_cpu*kv_size),
++            ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0);
++}
++
++ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
++    const int32_t ikv = map_layer_ids.at(il);
++    const auto & layer = layers[ikv];
++    GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split");
++    // M2's gate is `!v_trans && n_stream == 1` — when split is active the
++    // V layout is always non-transposed (matches get_v's non-split branch
++    // shape modulo head count). Same head-dim-1 stride pattern as K.
++    GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V");
++
++    auto * v = layer.v;
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++    const uint32_t n_head_gpu    = layer.gpu_heads;
++    const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_v;
++
++    return ggml_view_4d(ctx, v,
++            n_embd_head_v, n_head_gpu, n_kv, ns,
++            ggml_row_size(v->type, n_embd_head_v),
++            ggml_row_size(v->type, n_embd_gpu),
++            ggml_row_size(v->type, n_embd_gpu*kv_size),
++            ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0);
++}
++
++ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
++    const int32_t ikv = map_layer_ids.at(il);
++    const auto & layer = layers[ikv];
++    GGML_ASSERT(layer.gpu_heads > 0 && layer.v_cpu && "get_v_cpu called without an active headinfer split");
++    GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V");
++
++    auto * vc = layer.v_cpu;
++    const uint64_t kv_size = get_size();
++    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++    const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
++    const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_v;
++
++    return ggml_view_4d(ctx, vc,
++            n_embd_head_v, n_head_cpu, n_kv, ns,
++            ggml_row_size(vc->type, n_embd_head_v),
++            ggml_row_size(vc->type, n_embd_cpu),
++            ggml_row_size(vc->type, n_embd_cpu*kv_size),
++            ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0);
++}
++
+ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {
+     GGML_UNUSED(sinfo);
+ 
+@@ -3107,6 +3217,31 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons
+     return kv->get_v(ctx, il, n_kv, sinfos[i_cur]);
+ }
+ 
++// opencoti F5 M3 neo — per-batch wrappers around the head-split accessors.
++ggml_tensor * llama_kv_cache_context::get_k_gpu(ggml_context * ctx, int32_t il) const {
++    return kv->get_k_gpu(ctx, il, n_kv, sinfos[i_cur]);
++}
++
++ggml_tensor * llama_kv_cache_context::get_v_gpu(ggml_context * ctx, int32_t il) const {
++    return kv->get_v_gpu(ctx, il, n_kv, sinfos[i_cur]);
++}
++
++ggml_tensor * llama_kv_cache_context::get_k_cpu(ggml_context * ctx, int32_t il) const {
++    return kv->get_k_cpu(ctx, il, n_kv, sinfos[i_cur]);
++}
++
++ggml_tensor * llama_kv_cache_context::get_v_cpu(ggml_context * ctx, int32_t il) const {
++    return kv->get_v_cpu(ctx, il, n_kv, sinfos[i_cur]);
++}
++
++bool llama_kv_cache_context::headinfer_split_active(int32_t il) const {
++    return kv->headinfer_split_active(il);
++}
++
++uint32_t llama_kv_cache_context::headinfer_gpu_heads(int32_t il) const {
++    return kv->headinfer_gpu_heads(il);
++}
++
+ ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const {
+     return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]);
+ }
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -217,6 +217,23 @@ public:
+     ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
+     ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
+ 
++    // opencoti F5 M3 neo — head-split-aware K/V accessors. Return separate
++    // views onto the GPU-resident and host-resident head subsets without the
++    // concat-reassembly that get_k/get_v perform. Built on the SAME view
++    // machinery (see llama-kv-cache.cpp get_k for the dimensions); callers
++    // are expected to gate via headinfer_split_active(il) and to build their
++    // own combiner (M3 NEO builds two flash_attn_ext ops + a head-axis concat
++    // on the small attention outputs, eliminating M2's O(n_kv) stream-back).
++    // For non-split layers these return the equivalent of the un-split view
++    // for the "_gpu" half and nullptr for "_cpu" — callers must check
++    // headinfer_split_active first.
++    ggml_tensor * get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
++    ggml_tensor * get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
++    ggml_tensor * get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
++    ggml_tensor * get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
++    bool          headinfer_split_active(int32_t il) const;
++    uint32_t      headinfer_gpu_heads   (int32_t il) const;
++
+     // store k_cur and v_cur in the cache based on the provided head location
+     ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const;
+     ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const;
+@@ -444,6 +461,16 @@ public:
+     ggml_tensor * get_k(ggml_context * ctx, int32_t il) const;
+     ggml_tensor * get_v(ggml_context * ctx, int32_t il) const;
+ 
++    // opencoti F5 M3 neo — head-split-aware K/V accessors. Per-batch wrappers
++    // that delegate to the underlying llama_kv_cache with the current n_kv /
++    // slot info. See llama-kv-cache.h for semantics.
++    ggml_tensor * get_k_gpu(ggml_context * ctx, int32_t il) const;
++    ggml_tensor * get_v_gpu(ggml_context * ctx, int32_t il) const;
++    ggml_tensor * get_k_cpu(ggml_context * ctx, int32_t il) const;
++    ggml_tensor * get_v_cpu(ggml_context * ctx, int32_t il) const;
++    bool          headinfer_split_active(int32_t il) const;
++    uint32_t      headinfer_gpu_heads   (int32_t il) const;
++
+     // store k_cur and v_cur in the cache based on the provided head location
+     // note: the heads in k_cur and v_cur should be laid out contiguously in memory
+     //   - k_cur  [n_embd_head_k, n_head_k, n_tokens]
+diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh
+--- a/llamafile/build-functions.sh
++++ b/llamafile/build-functions.sh
+@@ -218,10 +218,14 @@ compile_ggml_core() {
+     local llama_cpp_dir="$1"
+     local build_dir="$2"
+ 
++    # opencoti F5 M3-B: ggml-neo-pipeline.cpp added so the CUDA backend's
++    # graph_compute hook can call the orchestrator API (registered cur_g
++    # detection + event slot get/set + dispatch-count increment).
+     local ggml_core_sources="\
+         $llama_cpp_dir/ggml/src/ggml.c \
+         $llama_cpp_dir/ggml/src/ggml-alloc.c \
+         $llama_cpp_dir/ggml/src/ggml-backend.cpp \
++        $llama_cpp_dir/ggml/src/ggml-neo-pipeline.cpp \
+         $llama_cpp_dir/ggml/src/ggml-backend-meta.cpp \
+         $llama_cpp_dir/ggml/src/ggml-quants.c \
+         $llama_cpp_dir/ggml/src/ggml-threading.cpp"

patches/0031-per-stream-split.patch

@@ -0,0 +1,1513 @@
+From: opencoti
+Subject: [PATCH 0031] F4 M3 Phase 4 — per-stream KV tensor split (task #109)
+
+Resolves the long-deferred Phase 4 (`docs/decisions/0001-lazy-slot-context.md`)
+of F4 M3 — every layer's K/V (and M2's CPU subset) become OWNING per-stream
+tensor vectors rather than one 3-D tensor with a stream dim, with each
+(stream, buffer-type) pair owning its own backend buffer. Unified mode
+(`n_stream == 1`) collapses to the pre-Phase-4 layout and stays
+byte-identical; non-unified mode (`n_stream > 1`) is functional for the
+first time, unlocking the multi-tenant codepaths (M2 head-split now lifts
+its `n_stream == 1` construction-time gate).
+
+Sub-milestone shape (each landed against the M5-shipped tree at HEAD =
+`2373e69` and verified by the M5 stack + M0 turn-2 + M1 eviction + M2
+residency benches before composing):
+
+  - Phase 4-A — per-stream tensor split. `kv_layer` replaces
+    `k`/`v`/`k_cpu`/`v_cpu` direct pointers with
+    `std::vector<ggml_tensor *> k_per_stream` / `v_per_stream` /
+    `k_cpu_per_stream` / `v_cpu_per_stream`; each per-stream tensor is
+    `ggml_new_tensor_3d(ctx_s, type, n_embd, kv_size, 1)`. The trailing
+    `dim 3 == 1` keeps stride math identical to the pre-Phase-4
+    `[n_embd, kv_size, n_stream]` layout under unified mode. The
+    `k_stream` / `v_stream` 2-D view vectors are preserved (offset 0
+    within each per-stream tensor) so state I/O and cross-stream copies
+    keep operating on the existing view handles.
+
+  - Phase 4-B — per-stream backend buffers. `ctx_map` becomes a nested
+    `std::vector<std::map<ggml_backend_buffer_type_t, ggml_context_ptr,
+    ggml_backend_buft_comparator>> ctx_map_per_stream` indexed by stream,
+    with a `ctx_for_buft(buft, s)` lambda that returns the get-or-create
+    `(stream, buft)` ctx (mem_size budgeted at `4u*n_layer_kv*
+    ggml_tensor_overhead()`). Allocation loops nest stream-outer ×
+    buft-inner so each (stream, buft) pair allocates one backend buffer.
+    Phase 1 `ensure_cleared`, Phase 3 `opencoti_decommit_layer_range`,
+    and the M2 CPU subset construction follow the same per-stream
+    iteration shape.
+
+  - Phase 4-C — reworked write/read graph ops. `cpy_k`/`cpy_v` ship
+    option C-2 (N separate `ggml_set_rows`, one per `sinfo.n_stream()`
+    stream, tied via `ggml_add` on 1-D views — unified-mode batch_ns==1
+    collapses to one `set_rows`, byte-identical). C-1 (concat-then-
+    scatter) was rejected at impl time: `ggml_concat` materializes a
+    new buffer rather than viewing inputs, so writes routed through the
+    concat result would not land in the per-stream tensors. `get_k`/
+    `get_v` build per-cache-stream 4-D views and `ggml_concat` them
+    along dim 3 (stream axis) — unified mode yields a single view, no
+    concat. `set_input_k_idxs`/`set_input_v_idxs` emit LOCAL per-stream
+    indices unconditionally (Phase 4-F fix; the previous gating on
+    `sinfo.n_stream() > 1` left global indices in --parallel N servers
+    serving single-session batches and crashed CUDA decode against the
+    per-stream destination). `build_graph_shift` loops streams inside
+    the layer loop, slicing `inp->k_shift` per stream.
+
+  - Phase 4-D — M2 head-split is stream-aware. `get_k_gpu`/`get_v_gpu`/
+    `get_k_cpu`/`get_v_cpu` (and the M2 split branch inside `get_k`/
+    `get_v`/`cpy_k`/`cpy_v`) build per-stream 4-D views and concat
+    them on dim 3 (stream axis); the construction-time gate at
+    `headinfer_gpu_heads_frac < 1.0 && offload && !v_trans && n_stream
+    == 1` drops the `n_stream == 1` clause so per-stream CPU subsets
+    actually allocate for non-unified mode.
+
+  - Phase 4-E — cross-stream copies in `update()`, `state_write_data`,
+    `state_read_data`, and `seq_key_scores` continue to operate on the
+    existing `k_stream[s]` / `v_stream[s]` 2-D views (now offset-0 views
+    of per-stream tensors). Mechanically distinct from pre-Phase-4 (one
+    buffer per stream rather than one shared buffer), semantically the
+    same per the buffer transfer mechanism.
+
+  - Phase 4-F — verification gates. Cosine ≥ 0.999 unified-mode
+    equivalence captured indirectly via the M5 stack bench (B/A0/A1/
+    A2/A3/S configs all produce byte-identical KV MiB and
+    `t2_cached_n=1271` against the M5-shipped baseline) plus M2's
+    explicit greedy-decode byte-equality check (`split text ===
+    baseline text`, 6/6 PASS). Multi-stream RSS exercise comes via the
+    M0 turn-2 scenarioB (`--parallel 2` with two sessions sharing a
+    large system prefix — 5/5 PASS, validating per-stream KV reuse).
+    M3 neo-pipeline O off-identity config crashes on both pre-Phase-4
+    and post-Phase-4 binaries with `concat.cu:165 GGML_ASSERT(src0->
+    type == GGML_TYPE_F32)` — pre-existing M3 bug, DEFERRED as task
+    #278 and recorded as bug-207. Hook count stable at 20.
+
+The HARD gates that survived from earlier sub-milestones (the
+`GGML_ASSERT(n_stream == 1 && "Phase 4-A: get_k temporarily gated...";
+"Phase 4-C lifts")` shape on get_k/get_v/cpy_k/cpy_v/build_graph_shift,
+and the parallel `Phase 4-D lifts` shape on the M2 accessors) all reach
+ship as lifted. Five `GGML_ASSERT(n_stream == 1)` lines deleted; the
+remaining `n_stream == 1` checks are real preconditions (e.g., v_trans +
+multi-stream is asserted out in cpy_v as not-a-real-workload).
+
+Unblocks F5 M6 (PolyKV) which needs per-stream lifetimes cleanly
+expressed for the shared compressed KV pool.
+
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -160,14 +160,35 @@ llama_kv_cache::llama_kv_cache(
+             return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;
+         }
+     };
+-    std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator> ctx_map;
+ 
+-    // create a context for each buffer type
+-    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {
++    // opencoti F4 M3 Phase 4-B — per-stream backend buffer ownership.
++    // Each stream gets its own map of (buft → ctx) so each (stream, buft)
++    // pair allocates its own backend buffer. Phase 3's
++    // opencoti_decommit_layer_range then collapses to "decommit this
++    // stream's buffer range" cleanly, and M6 PolyKV's per-stream
++    // lifetimes (enroll/disenroll a stream into the pool) become
++    // mechanically straightforward.
++    //
++    // Unified mode (n_stream == 1): one stream, one inner map per
++    // buft, byte-identical layout to pre-Phase-4 (the buffer-allocation
++    // loop below ends up with exactly the same `(ctx, buf)` pairs in
++    // ctxs_bufs as before, in the same order, with the same sizes).
++    std::vector<std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator>> ctx_map_per_stream;
++    ctx_map_per_stream.resize(n_stream);
++
++    // ctx_for_buft(buft, s): get-or-create the ggml_context for
++    // (stream s, buft). Each per-stream ctx holds at most 4 tensors
++    // per layer (K + V owning, K + V views; doubled when M2's CPU
++    // subset is engaged on a layer). Budget conservatively at
++    // 4*n_layer_kv slots per stream per buft — address space is free,
++    // RSS still grows only on first write.
++    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft, uint32_t s) -> ggml_context * {
++        GGML_ASSERT(s < ctx_map_per_stream.size());
++        auto & ctx_map = ctx_map_per_stream[s];
+         auto it = ctx_map.find(buft);
+         if (it == ctx_map.end()) {
+             ggml_init_params params = {
+-                /*.mem_size   =*/ size_t(2u*(1 + n_stream)*n_layer_kv*ggml_tensor_overhead()),
++                /*.mem_size   =*/ size_t(4u*n_layer_kv*ggml_tensor_overhead()),
+                 /*.mem_buffer =*/ NULL,
+                 /*.no_alloc   =*/ true,
+             };
+@@ -255,10 +276,10 @@ llama_kv_cache::llama_kv_cache(
+ 
+         LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name);
+ 
+-        ggml_context * ctx = ctx_for_buft(buft);
+-        if (!ctx) {
+-            throw std::runtime_error("failed to create ggml context for kv cache");
+-        }
++        // opencoti F4 M3 Phase 4-B: per-stream ctx selection has moved
++        // into the per-stream loop below. The layer-level `buft` is the
++        // device buffer-type for this layer; each stream allocates its
++        // own ctx for (buft, stream) via ctx_for_buft(buft, s).
+ 
+         const bool has_k = true;
+         const bool has_v = !is_mla;
+@@ -267,14 +288,21 @@ llama_kv_cache::llama_kv_cache(
+         // Head-residency split: keep the first gpu_heads KV heads on this
+         // (offloaded/device) layer's buffer and put the remaining heads on a
+         // host buffer. Splitting on KV-head boundaries is inherently GQA-safe.
+-        // Gated to offloaded, non-transposed-V, unified (n_stream==1) layers;
+-        // frac >= 1.0 (or <= 1 head) means no split (byte-identical upstream).
++        // Gated to offloaded, non-transposed-V layers; frac >= 1.0 (or <= 1
++        // head) means no split (byte-identical upstream).
++        //
++        // opencoti F4 M3 Phase 4-D: lifted the historical n_stream == 1
++        // gate. With per-stream owning tensors and per-stream backend
++        // buffers (Phase 4-A/B), each stream's GPU/CPU half lives in its
++        // own buffer, so the split is well-defined for n_stream > 1.
++        // Unified mode (n_stream == 1) collapses to the pre-Phase-4
++        // single-stream layout, byte-identical.
+         uint32_t gpu_heads    = 0;
+         uint32_t n_embd_k_gpu = n_embd_k_gqa;
+         uint32_t n_embd_v_gpu = n_embd_v_gqa;
+         uint32_t n_embd_k_cpu = 0;
+         uint32_t n_embd_v_cpu = 0;
+-        if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans && n_stream == 1) {
++        if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans) {
+             const uint32_t n_head_kv     = hparams.n_head_kv(il);
+             const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
+             const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
+@@ -291,39 +319,119 @@ llama_kv_cache::llama_kv_cache(
+             }
+         }
+ 
+-        ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gpu, kv_size, n_stream) : nullptr;
+-        ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gpu, kv_size, n_stream) : nullptr;
+-
+-        has_k && ggml_format_name(k, "cache_k_l%d", il);
+-        has_v && ggml_format_name(v, "cache_v_l%d", il);
+-
+-        // opencoti F5 M2 headinfer — the host-resident head subset lives in the
+-        // CPU buffer-type context; the buffer-allocation loop below picks it up
+-        // automatically (ctx_for_buft keys contexts by buffer type).
+-        ggml_tensor * k_cpu = nullptr;
+-        ggml_tensor * v_cpu = nullptr;
+-        if (gpu_heads > 0) {
+-            ggml_context * ctx_cpu = ctx_for_buft(ggml_backend_cpu_buffer_type());
+-            if (!ctx_cpu) {
+-                throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache");
+-            }
+-            k_cpu = has_k ? ggml_new_tensor_3d(ctx_cpu, type_k, n_embd_k_cpu, kv_size, n_stream) : nullptr;
+-            v_cpu = has_v ? ggml_new_tensor_3d(ctx_cpu, type_v, n_embd_v_cpu, kv_size, n_stream) : nullptr;
+-            has_k && k_cpu && ggml_format_name(k_cpu, "cache_k_cpu_l%d", il);
+-            has_v && v_cpu && ggml_format_name(v_cpu, "cache_v_cpu_l%d", il);
+-        }
+-
++        // opencoti F4 M3 Phase 4 — per-stream tensor split.
++        // Each stream gets its own OWNING 3-D tensor of shape
++        // [n_embd, kv_size, 1]. Dim 3 == 1 keeps stride math
++        // identical to the pre-Phase-4 single-tensor layout, so the
++        // unified-mode (n_stream == 1) buffer layout is byte-identical
++        // to pre-Phase-4. Phase 4-B: each stream also gets its own
++        // (buft → ctx) entry, so each (stream, buft) pair allocates
++        // its own backend buffer in the loop below.
++        std::vector<ggml_tensor *> k_per_stream;
++        std::vector<ggml_tensor *> v_per_stream;
++        std::vector<ggml_tensor *> k_cpu_per_stream;
++        std::vector<ggml_tensor *> v_cpu_per_stream;
+         std::vector<ggml_tensor *> k_stream;
+         std::vector<ggml_tensor *> v_stream;
+ 
+         for (uint32_t s = 0; s < n_stream; ++s) {
+-            k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gpu, kv_size, k->nb[1], s*k->nb[2]) : nullptr);
+-            v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gpu, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
++            ggml_context * ctx_s = ctx_for_buft(buft, s);
++            if (!ctx_s) {
++                throw std::runtime_error("failed to create ggml context for kv cache");
++            }
++
++            ggml_tensor * k_s = has_k
++                ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, kv_size, 1)
++                : nullptr;
++            ggml_tensor * v_s = has_v
++                ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, kv_size, 1)
++                : nullptr;
++
++            // Tensor naming preserves the pre-Phase-4 name in unified
++            // mode (no `_s%u` suffix) so external state-IO callers that
++            // grep by name find an unchanged layout.
++            if (k_s) {
++                if (n_stream == 1) {
++                    ggml_format_name(k_s, "cache_k_l%d", il);
++                } else {
++                    ggml_format_name(k_s, "cache_k_l%d_s%u", il, s);
++                }
++            }
++            if (v_s) {
++                if (n_stream == 1) {
++                    ggml_format_name(v_s, "cache_v_l%d", il);
++                } else {
++                    ggml_format_name(v_s, "cache_v_l%d_s%u", il, s);
++                }
++            }
++
++            k_per_stream.push_back(k_s);
++            v_per_stream.push_back(v_s);
++
++            // opencoti F5 M2 headinfer — per-stream host-resident head
++            // subset. Per-stream ctx_cpu so each stream's CPU half lives
++            // in its own host buffer (Phase 4-B). Vectors stay empty
++            // when gpu_heads == 0 so headinfer_split_active() can test
++            // by vector size.
++            if (gpu_heads > 0) {
++                ggml_context * ctx_cpu_s = ctx_for_buft(ggml_backend_cpu_buffer_type(), s);
++                if (!ctx_cpu_s) {
++                    throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache");
++                }
++                ggml_tensor * k_cpu_s = has_k
++                    ? ggml_new_tensor_3d(ctx_cpu_s, type_k, n_embd_k_cpu, kv_size, 1)
++                    : nullptr;
++                ggml_tensor * v_cpu_s = has_v
++                    ? ggml_new_tensor_3d(ctx_cpu_s, type_v, n_embd_v_cpu, kv_size, 1)
++                    : nullptr;
++                if (k_cpu_s) {
++                    if (n_stream == 1) {
++                        ggml_format_name(k_cpu_s, "cache_k_cpu_l%d", il);
++                    } else {
++                        ggml_format_name(k_cpu_s, "cache_k_cpu_l%d_s%u", il, s);
++                    }
++                }
++                if (v_cpu_s) {
++                    if (n_stream == 1) {
++                        ggml_format_name(v_cpu_s, "cache_v_cpu_l%d", il);
++                    } else {
++                        ggml_format_name(v_cpu_s, "cache_v_cpu_l%d_s%u", il, s);
++                    }
++                }
++                k_cpu_per_stream.push_back(k_cpu_s);
++                v_cpu_per_stream.push_back(v_cpu_s);
++            }
++
++            // 2-D views into per-stream 3-D tensor. Offset is 0 within
++            // the owning tensor — each stream owns the whole 3-D extent.
++            //
++            // opencoti F4 M3 Phase 4-E: keeping k_stream/v_stream as the
++            // existing 2-D view vectors means cross-stream copies in
++            // update() (ggml_backend_tensor_copy between views), state
++            // I/O paths (ggml_backend_tensor_set on the per-stream view),
++            // and seq_key_scores all work without per-stream logic at
++            // the consumer site — the view routes through to the
++            // per-stream buffer transparently. Unified-mode operations
++            // are byte-identical to pre-Phase-4 because the view has
++            // identical ne/nb and (in unified mode) the same underlying
++            // buffer layout.
++            k_stream.push_back(k_s
++                ? ggml_view_2d(ctx_s, k_s, n_embd_k_gpu, kv_size, k_s->nb[1], 0)
++                : nullptr);
++            v_stream.push_back(v_s
++                ? ggml_view_2d(ctx_s, v_s, n_embd_v_gpu, kv_size, v_s->nb[1], 0)
++                : nullptr);
+         }
+ 
+         map_layer_ids[il] = layers.size();
+ 
+-        layers.push_back({ il, k, v, k_stream, v_stream, k_cpu, v_cpu, gpu_heads });
++        layers.push_back({
++            il,
++            k_per_stream, v_per_stream,
++            k_stream, v_stream,
++            k_cpu_per_stream, v_cpu_per_stream,
++            gpu_heads,
++        });
+     }
+ 
+     if (reuse) {
+@@ -359,31 +467,46 @@ llama_kv_cache::llama_kv_cache(
+     // scheduler-visible tensor->buffer wiring intact (sched_reserve runs
+     // at construction time and asserts buffer_id >= 0) while moving the
+     // RSS commit to the first ensure_cleared() call.
+-    for (auto & [buft, ctx] : ctx_map) {
+-        ggml_backend_buffer_t buf;
+-        if (model.hparams.no_alloc) {
+-            buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+-            for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+-                t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
++    //
++    // opencoti F4 M3 Phase 4-B: per-stream backend buffers. Each
++    // (stream, buft) pair allocates its own backend buffer here.
++    // Iteration order is stable: outer loop over streams (s=0,1,...),
++    // inner loop over bufts (sorted by name via the ctx_map comparator).
++    // In unified mode (n_stream == 1) this collapses to exactly one
++    // pass per buft — byte-identical to pre-Phase-4 ctxs_bufs layout.
++    for (uint32_t s = 0; s < n_stream; ++s) {
++        for (auto & [buft, ctx] : ctx_map_per_stream[s]) {
++            ggml_backend_buffer_t buf;
++            if (model.hparams.no_alloc) {
++                buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
++                for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
++                    t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
++                }
++            } else {
++                buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet)
++            }
++            if (!buf) {
++                throw std::runtime_error("failed to allocate buffer for kv cache");
++            }
++
++            if (n_stream == 1) {
++                LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__,
++                        ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
++            } else {
++                LLAMA_LOG_INFO("%s: %10s KV buffer (stream %u) size = %8.2f MiB\n", __func__,
++                        ggml_backend_buffer_name(buf), s, ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+             }
+-        } else {
+-            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet)
+-        }
+-        if (!buf) {
+-            throw std::runtime_error("failed to allocate buffer for kv cache");
+-        }
+ 
+-        LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
++            if (model.hparams.no_alloc) {
++                // Dummy buffer is size 0; clearing is a no-op but keeps the
++                // upstream invariant that constructed buffers are zeroed.
++                ggml_backend_buffer_clear(buf, 0);
++            }
++            // else: leave the buffer un-cleared. ensure_cleared() will zero it
++            // on the first inference entry point that touches tensor data.
+ 
+-        if (model.hparams.no_alloc) {
+-            // Dummy buffer is size 0; clearing is a no-op but keeps the
+-            // upstream invariant that constructed buffers are zeroed.
+-            ggml_backend_buffer_clear(buf, 0);
++            ctxs_bufs.emplace_back(std::move(ctx), buf);
+         }
+-        // else: leave the buffer un-cleared. ensure_cleared() will zero it
+-        // on the first inference entry point that touches tensor data.
+-
+-        ctxs_bufs.emplace_back(std::move(ctx), buf);
+     }
+ 
+     if (model.hparams.no_alloc) {
+@@ -471,33 +594,43 @@ void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) {
+     // memset on the malloc'd buffer; for offloaded buffers it's a backend
+     // op. Either way only bytes for cells [a, b) get touched, so on Linux
+     // only those pages get committed to RSS.
++    //
++    // opencoti F4 M3 Phase 4: per-stream tensors. Each layer's
++    // k_per_stream[s] / v_per_stream[s] is now its own owning tensor
++    // (shape [n_embd, kv_size, 1]) so the per-stream stride math
++    // collapses to a plain cell-offset memset within each stream's
++    // tensor. M2's CPU heads follow the same pattern via
++    // k_cpu_per_stream / v_cpu_per_stream (empty when no split).
+     for (const auto & layer : layers) {
+         for (uint32_t st = 0; st < n_stream; ++st) {
+-            if (layer.k) {
+-                const size_t off = (size_t) st * (size_t) layer.k->nb[2]
+-                                 + (size_t) a  * (size_t) layer.k->nb[1];
+-                const size_t sz  = (size_t) (b - a) * (size_t) layer.k->nb[1];
+-                ggml_backend_tensor_memset(layer.k, 0, off, sz);
+-            }
+-            if (layer.v) {
+-                const size_t off = (size_t) st * (size_t) layer.v->nb[2]
+-                                 + (size_t) a  * (size_t) layer.v->nb[1];
+-                const size_t sz  = (size_t) (b - a) * (size_t) layer.v->nb[1];
+-                ggml_backend_tensor_memset(layer.v, 0, off, sz);
+-            }
+-            // opencoti F5 M2 headinfer — clear the host-resident head subset
+-            // too (its own strides; same [a, b) cell range).
+-            if (layer.k_cpu) {
+-                const size_t off = (size_t) st * (size_t) layer.k_cpu->nb[2]
+-                                 + (size_t) a  * (size_t) layer.k_cpu->nb[1];
+-                const size_t sz  = (size_t) (b - a) * (size_t) layer.k_cpu->nb[1];
+-                ggml_backend_tensor_memset(layer.k_cpu, 0, off, sz);
++            auto * k = layer.k_per_stream[st];
++            if (k) {
++                const size_t off = (size_t) a       * (size_t) k->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) k->nb[1];
++                ggml_backend_tensor_memset(k, 0, off, sz);
++            }
++            auto * v = layer.v_per_stream[st];
++            if (v) {
++                const size_t off = (size_t) a       * (size_t) v->nb[1];
++                const size_t sz  = (size_t) (b - a) * (size_t) v->nb[1];
++                ggml_backend_tensor_memset(v, 0, off, sz);
++            }
++            // M2 headinfer host-resident head subset (same [a, b) cells).
++            if (st < layer.k_cpu_per_stream.size()) {
++                auto * k_cpu = layer.k_cpu_per_stream[st];
++                if (k_cpu) {
++                    const size_t off = (size_t) a       * (size_t) k_cpu->nb[1];
++                    const size_t sz  = (size_t) (b - a) * (size_t) k_cpu->nb[1];
++                    ggml_backend_tensor_memset(k_cpu, 0, off, sz);
++                }
+             }
+-            if (layer.v_cpu) {
+-                const size_t off = (size_t) st * (size_t) layer.v_cpu->nb[2]
+-                                 + (size_t) a  * (size_t) layer.v_cpu->nb[1];
+-                const size_t sz  = (size_t) (b - a) * (size_t) layer.v_cpu->nb[1];
+-                ggml_backend_tensor_memset(layer.v_cpu, 0, off, sz);
++            if (st < layer.v_cpu_per_stream.size()) {
++                auto * v_cpu = layer.v_cpu_per_stream[st];
++                if (v_cpu) {
++                    const size_t off = (size_t) a       * (size_t) v_cpu->nb[1];
++                    const size_t sz  = (size_t) (b - a) * (size_t) v_cpu->nb[1];
++                    ggml_backend_tensor_memset(v_cpu, 0, off, sz);
++                }
+             }
+         }
+     }
+@@ -542,22 +675,25 @@ bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) {
+ }
+ 
+ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+-// Decommit pages of every layer's K/V tensors in the byte-range that
+-// corresponds to cells [from_cells, to_cells). Returns those pages to the OS
+-// — RSS drops by the size of the range, and the next access to those pages
+-// will fault them back in as fresh zero-filled pages.
++// Decommit pages of one layer's K/V tensor for ONE STREAM in the byte-
++// range that corresponds to cells [from_cells, to_cells). Returns those
++// pages to the OS — RSS drops by the size of the range, and the next
++// access to those pages will fault them back in as fresh zero-filled
++// pages.
+ //
+ // Only safe to call after kv_size_alloc has been reset to from_cells (or
+-// lower) and n_cells_cleared has been reset accordingly, so the bookkeeping
+-// stays in sync with what's actually backed by physical memory.
++// lower) and n_cells_cleared has been reset accordingly, so the
++// bookkeeping stays in sync with what's actually backed by physical
++// memory.
+ //
+-// The decommit is per-stream because the KV tensors are 3-D
+-// [n_embd, kv_size, n_stream]; cells [a, b) within stream s occupy a
+-// contiguous byte-range whose start address is (base + s*nb[2] + a*nb[1])
+-// and length is (b-a)*nb[1]. posix_madvise is page-grained — bytes that
+-// don't cover a full page are silently kept resident, which is fine for
+-// the cells that straddle page boundaries.
+-static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream,
++// opencoti F4 M3 Phase 4: per-stream tensors. `t` is now ONE stream's
++// owning 3-D tensor (dim 3 == 1). Cells [a, b) occupy contiguous bytes
++// at (base + a*nb[1]) for length (b-a)*nb[1]. The stream-stride loop
++// of the pre-Phase-4 implementation collapses; callers iterate streams
++// at the call site instead. posix_madvise is page-grained — bytes that
++// don't cover a full page are silently kept resident, which is fine
++// for the cells that straddle page boundaries.
++static void opencoti_decommit_layer_range(ggml_tensor * t,
+                                           uint32_t from_cells, uint32_t to_cells) {
+     if (!t || to_cells <= from_cells) {
+         return;
+@@ -579,37 +715,32 @@ static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream,
+     if (!base) {
+         return;
+     }
+-    const size_t stride_stream = (size_t) t->nb[2];
+-    const size_t stride_cell   = (size_t) t->nb[1];
+-    const size_t off_in_buf    = (size_t) ((char *) t->data - (char *) base);
+-    for (uint32_t s = 0; s < n_stream; ++s) {
+-        const size_t off = off_in_buf
+-                         + (size_t) s          * stride_stream
+-                         + (size_t) from_cells * stride_cell;
+-        const size_t sz  = (size_t) (to_cells - from_cells) * stride_cell;
+-        // posix_madvise wants the address — work in absolute terms.
+-        void * addr = (char *) base + off;
+-        // Round up to a page boundary on the low side, round down on the
+-        // high side, so we only decommit pages we fully own. The cell-aligned
+-        // address is unlikely to be page-aligned in general.
+-        const size_t page = (size_t) sysconf(_SC_PAGESIZE);
+-        if (page == 0 || page == (size_t) -1) {
+-            return;
+-        }
+-        uintptr_t lo = (uintptr_t) addr;
+-        uintptr_t hi = lo + sz;
+-        uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1);
+-        uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1);
+-        if (hi_aligned <= lo_aligned) {
+-            continue;
+-        }
+-        // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with
+-        // errno set). We treat any failure as advisory — the RSS win may
+-        // be skipped, but correctness is unaffected (the K/V buffer is
+-        // still mapped, just possibly still resident).
+-        (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned),
+-                       MADV_DONTNEED);
++    const size_t stride_cell = (size_t) t->nb[1];
++    const size_t off_in_buf  = (size_t) ((char *) t->data - (char *) base);
++    const size_t off = off_in_buf + (size_t) from_cells * stride_cell;
++    const size_t sz  = (size_t) (to_cells - from_cells) * stride_cell;
++    // posix_madvise wants the address — work in absolute terms.
++    void * addr = (char *) base + off;
++    // Round up to a page boundary on the low side, round down on the
++    // high side, so we only decommit pages we fully own. The cell-aligned
++    // address is unlikely to be page-aligned in general.
++    const size_t page = (size_t) sysconf(_SC_PAGESIZE);
++    if (page == 0 || page == (size_t) -1) {
++        return;
+     }
++    uintptr_t lo = (uintptr_t) addr;
++    uintptr_t hi = lo + sz;
++    uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1);
++    uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1);
++    if (hi_aligned <= lo_aligned) {
++        return;
++    }
++    // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with
++    // errno set). We treat any failure as advisory — the RSS win may
++    // be skipped, but correctness is unaffected (the K/V buffer is
++    // still mapped, just possibly still resident).
++    (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned),
++                   MADV_DONTNEED);
+ }
+ 
+ void llama_kv_cache::shrink_if_idle() {
+@@ -667,13 +798,20 @@ void llama_kv_cache::shrink_if_idle() {
+             (unsigned long long) (now - last_active_ms_), evicted);
+ 
+     for (const auto & layer : layers) {
+-        opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells);
+-        opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells);
+-        // opencoti F5 M2 headinfer — the host-resident head subset IS
+-        // host-pageable, so this is where the GPU-mode shrink actually returns
+-        // RSS (the device k/v above are skipped by the is_host() guard).
+-        opencoti_decommit_layer_range(layer.k_cpu, n_stream, from_cells, to_cells);
+-        opencoti_decommit_layer_range(layer.v_cpu, n_stream, from_cells, to_cells);
++        for (uint32_t s = 0; s < n_stream; ++s) {
++            opencoti_decommit_layer_range(layer.k_per_stream[s], from_cells, to_cells);
++            opencoti_decommit_layer_range(layer.v_per_stream[s], from_cells, to_cells);
++            // opencoti F5 M2 headinfer — the host-resident head subset
++            // IS host-pageable, so this is where the GPU-mode shrink
++            // actually returns RSS (the device k/v above are skipped
++            // by the is_host() guard in the function).
++            if (s < layer.k_cpu_per_stream.size()) {
++                opencoti_decommit_layer_range(layer.k_cpu_per_stream[s], from_cells, to_cells);
++            }
++            if (s < layer.v_cpu_per_stream.size()) {
++                opencoti_decommit_layer_range(layer.v_cpu_per_stream[s], from_cells, to_cells);
++            }
++        }
+     }
+ 
+     // After decommit, treat cells [from_cells, to_cells) as "uncleared" —
+@@ -1683,11 +1821,14 @@ bool llama_kv_cache::get_has_shift() const {
+ }
+ 
+ ggml_type llama_kv_cache::type_k() const {
+-    return layers[0].k->type;
++    // opencoti F4 M3 Phase 4: all per-stream tensors of a layer carry the
++    // same type — read it from stream 0. Works in both unified and
++    // non-unified modes.
++    return layers[0].k_per_stream[0]->type;
+ }
+ 
+ ggml_type llama_kv_cache::type_v() const {
+-    return layers[0].v->type;
++    return layers[0].v_per_stream[0]->type;
+ }
+ 
+ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
+@@ -1710,104 +1851,166 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+     const auto & layer = layers[ikv];
+-    auto * k = layer.k;
+ 
+     const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
+ 
+     // opencoti F5 M2 headinfer — reassemble the full head set from the
+-    // GPU-resident (k) and host-resident (k_cpu) subsets by concat on the head
++    // GPU-resident and host-resident subsets by concat on the head
+     // dimension. The backend scheduler streams k_cpu to the compute backend.
+-    if (layer.gpu_heads > 0 && layer.k_cpu) {
++    //
++    // opencoti F4 M3 Phase 4-D: stream-aware. For each cache stream covered
++    // by sinfo, build a 4-D head-axis-concat view (GPU half + CPU half)
++    // with that stream's per-stream tensors. Then concat the per-stream
++    // results along the stream axis (dim 3). Unified mode (sinfo.s1 ==
++    // sinfo.s0 == 0) collapses to one 4-D head-concat view of stream 0's
++    // tensors — byte-identical to pre-Phase-4.
++    if (layer.gpu_heads > 0 && !layer.k_cpu_per_stream.empty() && layer.k_cpu_per_stream[0]) {
+         const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
+         const uint32_t n_head_gpu    = layer.gpu_heads;
+         const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
+         const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_k;
+         const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_k;
+-        auto * kc = layer.k_cpu;
+-        ggml_tensor * kg_v = ggml_view_4d(ctx, k,
+-                n_embd_head_k, n_head_gpu, n_kv, ns,
+-                ggml_row_size(k->type, n_embd_head_k),
+-                ggml_row_size(k->type, n_embd_gpu),
+-                ggml_row_size(k->type, n_embd_gpu*kv_size),
+-                ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0);
+-        ggml_tensor * kc_v = ggml_view_4d(ctx, kc,
+-                n_embd_head_k, n_head_cpu, n_kv, ns,
+-                ggml_row_size(kc->type, n_embd_head_k),
+-                ggml_row_size(kc->type, n_embd_cpu),
+-                ggml_row_size(kc->type, n_embd_cpu*kv_size),
+-                ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0);
+-        return ggml_concat(ctx, kg_v, kc_v, 1);
+-    }
+-
+-    const uint64_t n_embd_k_gqa = k->ne[0];
+ 
++        auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++            ggml_tensor * k_s  = layer.k_per_stream[s_cache];
++            ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache];
++            ggml_tensor * kg_v = ggml_view_4d(ctx, k_s,
++                    n_embd_head_k, n_head_gpu, n_kv, 1,
++                    ggml_row_size(k_s->type, n_embd_head_k),
++                    ggml_row_size(k_s->type, n_embd_gpu),
++                    ggml_row_size(k_s->type, n_embd_gpu*kv_size),
++                    0);
++            ggml_tensor * kc_v = ggml_view_4d(ctx, kc_s,
++                    n_embd_head_k, n_head_cpu, n_kv, 1,
++                    ggml_row_size(kc_s->type, n_embd_head_k),
++                    ggml_row_size(kc_s->type, n_embd_cpu),
++                    ggml_row_size(kc_s->type, n_embd_cpu*kv_size),
++                    0);
++            return ggml_concat(ctx, kg_v, kc_v, 1);
++        };
++
++        ggml_tensor * result = build_stream_view(sinfo.s0);
++        for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++            result = ggml_concat(ctx, result, build_stream_view(s), 3);
++        }
++        return result;
++    }
++
++    // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream axis
++    // (dim 3). In unified mode (sinfo.s1 == sinfo.s0 == 0) the loop does
++    // not fire, yielding a single 4-D view of layer.k_per_stream[0] —
++    // structurally and numerically byte-identical to pre-Phase-4 (same
++    // ne, same nb, same offset 0). In non-unified mode the result is N
++    // contiguous concat ops along dim 3 (the stream axis), one per
++    // covered cache stream. Cost: (ns - 1) concat nodes per layer per turn.
++    auto * k0 = layer.k_per_stream[sinfo.s0];
++    const uint64_t n_embd_k_gqa = k0->ne[0];
+     assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il));
+ 
+-    return ggml_view_4d(ctx, k,
+-            hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns,
+-            ggml_row_size(k->type, hparams.n_embd_head_k(il)),
+-            ggml_row_size(k->type, n_embd_k_gqa),
+-            ggml_row_size(k->type, n_embd_k_gqa*kv_size),
+-            ggml_row_size(k->type, n_embd_k_gqa*kv_size)*sinfo.s0);
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * k_s = layer.k_per_stream[s_cache];
++        return ggml_view_4d(ctx, k_s,
++                hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, 1,
++                ggml_row_size(k_s->type, hparams.n_embd_head_k(il)),
++                ggml_row_size(k_s->type, n_embd_k_gqa),
++                ggml_row_size(k_s->type, n_embd_k_gqa*kv_size),
++                0);
++    };
++
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
++    }
++    return result;
+ }
+ 
+ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+     const auto & layer = layers[ikv];
+-    auto * v = layer.v;
+ 
+     const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
+ 
+-    // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The split
+-    // is gated off when v_trans at construction, so this branch is always the
+-    // non-transposed layout (heads in dim 1, same as get_k).
+-    if (layer.gpu_heads > 0 && layer.v_cpu) {
++    // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The
++    // split is gated off when v_trans at construction, so this branch is
++    // always the non-transposed layout (heads in dim 1, same as get_k).
++    //
++    // opencoti F4 M3 Phase 4-D: stream-aware per-stream concat — same
++    // pattern as get_k's M2 branch.
++    if (layer.gpu_heads > 0 && !layer.v_cpu_per_stream.empty() && layer.v_cpu_per_stream[0]) {
+         const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
+         const uint32_t n_head_gpu    = layer.gpu_heads;
+         const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
+         const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_v;
+         const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_v;
+-        auto * vc = layer.v_cpu;
+-        ggml_tensor * vg_v = ggml_view_4d(ctx, v,
+-                n_embd_head_v, n_head_gpu, n_kv, ns,
+-                ggml_row_size(v->type, n_embd_head_v),
+-                ggml_row_size(v->type, n_embd_gpu),
+-                ggml_row_size(v->type, n_embd_gpu*kv_size),
+-                ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0);
+-        ggml_tensor * vc_v = ggml_view_4d(ctx, vc,
+-                n_embd_head_v, n_head_cpu, n_kv, ns,
+-                ggml_row_size(vc->type, n_embd_head_v),
+-                ggml_row_size(vc->type, n_embd_cpu),
+-                ggml_row_size(vc->type, n_embd_cpu*kv_size),
+-                ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0);
+-        return ggml_concat(ctx, vg_v, vc_v, 1);
+-    }
+-
+-    const uint64_t n_embd_v_gqa = v->ne[0];
++
++        auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++            ggml_tensor * v_s  = layer.v_per_stream[s_cache];
++            ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache];
++            ggml_tensor * vg_v = ggml_view_4d(ctx, v_s,
++                    n_embd_head_v, n_head_gpu, n_kv, 1,
++                    ggml_row_size(v_s->type, n_embd_head_v),
++                    ggml_row_size(v_s->type, n_embd_gpu),
++                    ggml_row_size(v_s->type, n_embd_gpu*kv_size),
++                    0);
++            ggml_tensor * vc_v = ggml_view_4d(ctx, vc_s,
++                    n_embd_head_v, n_head_cpu, n_kv, 1,
++                    ggml_row_size(vc_s->type, n_embd_head_v),
++                    ggml_row_size(vc_s->type, n_embd_cpu),
++                    ggml_row_size(vc_s->type, n_embd_cpu*kv_size),
++                    0);
++            return ggml_concat(ctx, vg_v, vc_v, 1);
++        };
++
++        ggml_tensor * result = build_stream_view(sinfo.s0);
++        for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++            result = ggml_concat(ctx, result, build_stream_view(s), 3);
++        }
++        return result;
++    }
++
++    // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream
++    // axis. Same shape as get_k, but with v_trans branching. Unified mode
++    // (sinfo.s1 == sinfo.s0 == 0) keeps a single 4-D view → byte-identical.
++    auto * v0 = layer.v_per_stream[sinfo.s0];
++    const uint64_t n_embd_v_gqa = v0->ne[0];
+ 
+     // [TAG_V_CACHE_VARIABLE]
+     assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il));
+ 
+     if (!v_trans) {
+         // note: v->nb[1] <= v->nb[2]
+-        return ggml_view_4d(ctx, v,
+-                hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, ns,
+-                ggml_row_size(v->type, hparams.n_embd_head_v(il)),          // v->nb[1]
+-                ggml_row_size(v->type, n_embd_v_gqa),                   // v->nb[2]
+-                ggml_row_size(v->type, n_embd_v_gqa*kv_size),           // v->nb[3]
+-                ggml_row_size(v->type, n_embd_v_gqa*kv_size)*sinfo.s0);
++        auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++            ggml_tensor * v_s = layer.v_per_stream[s_cache];
++            return ggml_view_4d(ctx, v_s,
++                    hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, 1,
++                    ggml_row_size(v_s->type, hparams.n_embd_head_v(il)),
++                    ggml_row_size(v_s->type, n_embd_v_gqa),
++                    ggml_row_size(v_s->type, n_embd_v_gqa*kv_size),
++                    0);
++        };
++        ggml_tensor * result = build_stream_view(sinfo.s0);
++        for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++            result = ggml_concat(ctx, result, build_stream_view(s), 3);
++        }
++        return result;
++    }
++
++    // v_trans branch: note v->nb[1] > v->nb[2]
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * v_s = layer.v_per_stream[s_cache];
++        return ggml_view_4d(ctx, v_s,
++                n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), 1,
++                ggml_row_size(v_s->type, kv_size*hparams.n_embd_head_v(il)),
++                ggml_row_size(v_s->type, kv_size),
++                ggml_row_size(v_s->type, kv_size*n_embd_v_gqa),
++                0);
++    };
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
+     }
+-
+-    // note: v->nb[1] > v->nb[2]
+-    return ggml_view_4d(ctx, v,
+-            n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), ns,
+-            ggml_row_size(v->type, kv_size*hparams.n_embd_head_v(il)),  // v->nb[1]
+-            ggml_row_size(v->type, kv_size),                        // v->nb[2]
+-            ggml_row_size(v->type, kv_size*n_embd_v_gqa),           // v->nb[3]
+-            ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0);
++    return result;
+ }
+ 
+ // opencoti F5 M3 neo — head-split-aware accessors. These factor the per-half
+@@ -1824,7 +2027,13 @@ bool llama_kv_cache::headinfer_split_active(int32_t il) const {
+         return false;
+     }
+     const auto & layer = layers[it->second];
+-    return layer.gpu_heads > 0 && layer.k_cpu != nullptr;
++    // opencoti F4 M3 Phase 4: per-stream k_cpu vector is empty when no split,
++    // populated when gpu_heads > 0. Stream 0 always exists in unified mode
++    // (the only mode where split engages today; the M2 construction gate
++    // requires n_stream == 1).
++    return layer.gpu_heads > 0
++        && !layer.k_cpu_per_stream.empty()
++        && layer.k_cpu_per_stream[0] != nullptr;
+ }
+ 
+ uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const {
+@@ -1840,93 +2049,131 @@ ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t
+     const auto & layer = layers[ikv];
+     GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split");
+ 
+-    auto * k = layer.k;
+-    const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    // opencoti F4 M3 Phase 4-D: per-stream M2 GPU subset. In unified
++    // mode (sinfo.s1 == sinfo.s0 == 0) the loop runs once and yields
++    // a single 4-D view of layer.k_per_stream[0]'s GPU half —
++    // byte-identical to pre-Phase-4 (same ne, same nb, offset 0).
++    const uint64_t kv_size       = get_size();
+     const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
+     const uint32_t n_head_gpu    = layer.gpu_heads;
+     const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_k;
+ 
+-    return ggml_view_4d(ctx, k,
+-            n_embd_head_k, n_head_gpu, n_kv, ns,
+-            ggml_row_size(k->type, n_embd_head_k),
+-            ggml_row_size(k->type, n_embd_gpu),
+-            ggml_row_size(k->type, n_embd_gpu*kv_size),
+-            ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0);
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * k_s = layer.k_per_stream[s_cache];
++        return ggml_view_4d(ctx, k_s,
++                n_embd_head_k, n_head_gpu, n_kv, 1,
++                ggml_row_size(k_s->type, n_embd_head_k),
++                ggml_row_size(k_s->type, n_embd_gpu),
++                ggml_row_size(k_s->type, n_embd_gpu*kv_size),
++                0);
++    };
++
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
++    }
++    return result;
+ }
+ 
+ ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+     const auto & layer = layers[ikv];
+-    GGML_ASSERT(layer.gpu_heads > 0 && layer.k_cpu && "get_k_cpu called without an active headinfer split");
+-
+-    auto * kc = layer.k_cpu;
+-    const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    GGML_ASSERT(layer.gpu_heads > 0
++                && !layer.k_cpu_per_stream.empty()
++                && layer.k_cpu_per_stream[0]
++                && "get_k_cpu called without an active headinfer split");
++
++    // opencoti F4 M3 Phase 4-D: per-stream M2 CPU subset. Same pattern
++    // as get_k_gpu; unified mode collapses to a single 4-D view.
++    const uint64_t kv_size       = get_size();
+     const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
+     const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
+     const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_k;
+ 
+-    return ggml_view_4d(ctx, kc,
+-            n_embd_head_k, n_head_cpu, n_kv, ns,
+-            ggml_row_size(kc->type, n_embd_head_k),
+-            ggml_row_size(kc->type, n_embd_cpu),
+-            ggml_row_size(kc->type, n_embd_cpu*kv_size),
+-            ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0);
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache];
++        return ggml_view_4d(ctx, kc_s,
++                n_embd_head_k, n_head_cpu, n_kv, 1,
++                ggml_row_size(kc_s->type, n_embd_head_k),
++                ggml_row_size(kc_s->type, n_embd_cpu),
++                ggml_row_size(kc_s->type, n_embd_cpu*kv_size),
++                0);
++    };
++
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
++    }
++    return result;
+ }
+ 
+ ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+     const auto & layer = layers[ikv];
+     GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split");
+-    // M2's gate is `!v_trans && n_stream == 1` — when split is active the
+-    // V layout is always non-transposed (matches get_v's non-split branch
+-    // shape modulo head count). Same head-dim-1 stride pattern as K.
++    // M2 requires non-transposed V (FA on). The split-active branch in
++    // get_v/cpy_v also requires this. Per-stream view + concat on stream
++    // axis. Phase 4-D: lifted the n_stream == 1 gate.
+     GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V");
+ 
+-    auto * v = layer.v;
+-    const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    const uint64_t kv_size       = get_size();
+     const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
+     const uint32_t n_head_gpu    = layer.gpu_heads;
+     const uint64_t n_embd_gpu    = (uint64_t) n_head_gpu * n_embd_head_v;
+ 
+-    return ggml_view_4d(ctx, v,
+-            n_embd_head_v, n_head_gpu, n_kv, ns,
+-            ggml_row_size(v->type, n_embd_head_v),
+-            ggml_row_size(v->type, n_embd_gpu),
+-            ggml_row_size(v->type, n_embd_gpu*kv_size),
+-            ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0);
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * v_s = layer.v_per_stream[s_cache];
++        return ggml_view_4d(ctx, v_s,
++                n_embd_head_v, n_head_gpu, n_kv, 1,
++                ggml_row_size(v_s->type, n_embd_head_v),
++                ggml_row_size(v_s->type, n_embd_gpu),
++                ggml_row_size(v_s->type, n_embd_gpu*kv_size),
++                0);
++    };
++
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
++    }
++    return result;
+ }
+ 
+ ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {
+     const int32_t ikv = map_layer_ids.at(il);
+     const auto & layer = layers[ikv];
+-    GGML_ASSERT(layer.gpu_heads > 0 && layer.v_cpu && "get_v_cpu called without an active headinfer split");
++    GGML_ASSERT(layer.gpu_heads > 0
++                && !layer.v_cpu_per_stream.empty()
++                && layer.v_cpu_per_stream[0]
++                && "get_v_cpu called without an active headinfer split");
+     GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V");
+ 
+-    auto * vc = layer.v_cpu;
+-    const uint64_t kv_size = get_size();
+-    const uint32_t ns      = sinfo.s1 - sinfo.s0 + 1;
++    // opencoti F4 M3 Phase 4-D: per-stream M2 CPU V subset.
++    const uint64_t kv_size       = get_size();
+     const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
+     const uint32_t n_head_cpu    = hparams.n_head_kv(il) - layer.gpu_heads;
+     const uint64_t n_embd_cpu    = (uint64_t) n_head_cpu * n_embd_head_v;
+ 
+-    return ggml_view_4d(ctx, vc,
+-            n_embd_head_v, n_head_cpu, n_kv, ns,
+-            ggml_row_size(vc->type, n_embd_head_v),
+-            ggml_row_size(vc->type, n_embd_cpu),
+-            ggml_row_size(vc->type, n_embd_cpu*kv_size),
+-            ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0);
++    auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * {
++        ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache];
++        return ggml_view_4d(ctx, vc_s,
++                n_embd_head_v, n_head_cpu, n_kv, 1,
++                ggml_row_size(vc_s->type, n_embd_head_v),
++                ggml_row_size(vc_s->type, n_embd_cpu),
++                ggml_row_size(vc_s->type, n_embd_cpu*kv_size),
++                0);
++    };
++
++    ggml_tensor * result = build_stream_view(sinfo.s0);
++    for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) {
++        result = ggml_concat(ctx, result, build_stream_view(s), 3);
++    }
++    return result;
+ }
+ 
+ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {
+-    GGML_UNUSED(sinfo);
+-
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+     const auto & layer = layers[ikv];
+-    ggml_tensor * k = layer.k;
+ 
+     const int64_t n_embd_head = k_cur->ne[0];
+     const int64_t n_head      = k_cur->ne[1];
+@@ -1941,46 +2188,116 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
+     // opencoti F5 M2 headinfer — scatter the incoming heads into the GPU and
+     // host subsets. The first gpu_heads heads (contiguous in k_cur's dim 0..1)
+     // store into k; the rest store into k_cpu, both at the same cell indices
+-    // (k_idxs). The single returned dep node ties both set_rows into the
+-    // caller's one ggml_build_forward_expand (its value is never read). Split
+-    // only happens for n_stream==1 (gated at construction) — no per-stream
+-    // reshape needed.
+-    if (layer.gpu_heads > 0 && layer.k_cpu) {
++    // (k_idxs). The single returned dep node ties all set_rows into the
++    // caller's one ggml_build_forward_expand (its value is never read).
++    //
++    // opencoti F4 M3 Phase 4-D: per-stream M2 scatter — for each batch
++    // stream s, slice k_cur and k_idxs by stream, split the head-axis
++    // slice into GPU + CPU halves, and emit 2 set_rows per stream against
++    // the per-stream tensors. In unified mode (sinfo.n_stream() == 1) this
++    // collapses to exactly 2 set_rows on stream 0's tensors — byte-identical
++    // to pre-Phase-4.
++    if (layer.gpu_heads > 0
++        && !layer.k_cpu_per_stream.empty()
++        && layer.k_cpu_per_stream[0]) {
+         const int64_t g = layer.gpu_heads;
+         const int64_t c = n_head - g;
+-        ggml_tensor * kc = layer.k_cpu;
+-        ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur, n_embd_head*g, n_tokens, k_cur->nb[2], 0);
+-        ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur, n_embd_head*c, n_tokens, k_cur->nb[2], (size_t) (g*k_cur->nb[1]));
+-        ggml_tensor * store_g = ggml_set_rows(ctx, k,  cur_g, k_idxs);
+-        ggml_tensor * store_c = ggml_set_rows(ctx, kc, cur_c, k_idxs);
+-        return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0));
++        const uint32_t batch_ns = sinfo.n_stream();
++        GGML_ASSERT(batch_ns >= 1);
++
++        const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns);
++        GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens);
++
++        ggml_tensor * tie = nullptr;
++        for (uint32_t s = 0; s < batch_ns; ++s) {
++            ggml_tensor * k_s  = layer.k_per_stream[sinfo.strm[s]];
++            ggml_tensor * kc_s = layer.k_cpu_per_stream[sinfo.strm[s]];
++
++            // k_cur is [head_dim, n_head, n_tokens]; nb[2] is the stride
++            // between tokens. The per-stream slice is the contiguous block
++            // of tokens_per_stream tokens at offset s*tokens_per_stream.
++            const size_t per_stream_off = (size_t) s * tokens_per_stream * k_cur->nb[2];
++            ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur,
++                    n_embd_head*g, tokens_per_stream,
++                    k_cur->nb[2], per_stream_off);
++            // CPU half: remaining c heads, offset by g heads (g rows in the head-merged dim 0)
++            ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur,
++                    n_embd_head*c, tokens_per_stream,
++                    k_cur->nb[2], per_stream_off + (size_t)(g*k_cur->nb[1]));
++            ggml_tensor * idxs_s = (batch_ns == 1)
++                ? k_idxs
++                : ggml_view_1d(ctx, k_idxs, tokens_per_stream,
++                        (size_t) s * tokens_per_stream * k_idxs->nb[0]);
++            ggml_tensor * store_g = ggml_set_rows(ctx, k_s,  cur_g, idxs_s);
++            ggml_tensor * store_c = ggml_set_rows(ctx, kc_s, cur_c, idxs_s);
++            ggml_tensor * pair = ggml_add(ctx,
++                    ggml_view_1d(ctx, store_g, 1, 0),
++                    ggml_view_1d(ctx, store_c, 1, 0));
++            if (!tie) {
++                tie = pair;
++            } else {
++                tie = ggml_add(ctx,
++                        ggml_view_1d(ctx, tie, 1, 0),
++                        ggml_view_1d(ctx, pair, 1, 0));
++            }
++        }
++        return tie;
+     }
+ 
+     k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0);
+ 
+-    const int64_t n_stream = k->ne[2];
+-
+-    if (n_stream > 1) {
+-        const int64_t kv_size = get_size();
+-
+-        assert(n_embd_gqa == k->ne[0]);
+-        assert(kv_size    == k->ne[1]);
+-
+-        // merge the buffer across all streams because the idxs are global
+-        k = ggml_reshape_2d(ctx, k, n_embd_gqa, kv_size*n_stream);
++    // opencoti F4 M3 Phase 4-C: per-stream scatter via N separate
++    // ggml_set_rows ops (option C-2). In unified mode (sinfo.n_stream()
++    // == 1) the fast path collapses to one ggml_set_rows against
++    // layer.k_per_stream[sinfo.strm[0]] — byte-identical to pre-Phase-4.
++    // In non-unified mode (sinfo.n_stream() > 1), set_input_k_idxs has
++    // emitted LOCAL indices (no global offset), so each per-stream slice
++    // of k_idxs is valid against its own k_per_stream[cache_strm] tensor.
++    // The N stores are tied into a single dep node via ggml_add so the
++    // caller's one ggml_build_forward_expand picks them all up.
++    //
++    // C-1 (concat-then-scatter) is not expressible with per-stream owning
++    // tensors in ggml: ggml_concat materializes a new buffer rather than
++    // creating a view into its inputs, so writes routed through the concat
++    // result would not land in the per-stream tensors. C-2 is the natural
++    // shape that preserves per-stream buffer ownership.
++    const uint32_t batch_ns = sinfo.n_stream();
++    GGML_ASSERT(batch_ns >= 1);
++
++    if (batch_ns == 1) {
++        ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[0]];
++        return ggml_set_rows(ctx, k_s, k_cur, k_idxs);
++    }
++
++    const int64_t tokens_per_stream = n_tokens / (int64_t) batch_ns;
++    GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens);
++
++    ggml_tensor * tie = nullptr;
++    for (uint32_t s = 0; s < batch_ns; ++s) {
++        ggml_tensor * k_s   = layer.k_per_stream[sinfo.strm[s]];
++        ggml_tensor * cur_s = ggml_view_2d(ctx, k_cur,
++                n_embd_gqa, tokens_per_stream,
++                k_cur->nb[1],
++                (size_t) s * tokens_per_stream * k_cur->nb[1]);
++        ggml_tensor * idxs_s = ggml_view_1d(ctx, k_idxs,
++                tokens_per_stream,
++                (size_t) s * tokens_per_stream * k_idxs->nb[0]);
++        ggml_tensor * store_s = ggml_set_rows(ctx, k_s, cur_s, idxs_s);
++        if (!tie) {
++            tie = store_s;
++        } else {
++            tie = ggml_add(ctx,
++                    ggml_view_1d(ctx, tie, 1, 0),
++                    ggml_view_1d(ctx, store_s, 1, 0));
++        }
+     }
+-
+-    // store the current K values into the cache
+-    return ggml_set_rows(ctx, k, k_cur, k_idxs);
++    return tie;
+ }
+ 
+ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const {
+-    GGML_UNUSED(sinfo);
+-
+     const int32_t ikv = map_layer_ids.at(il);
+ 
+     const auto & layer = layers[ikv];
+-    auto * v = layer.v;
+ 
+     const int64_t n_embd_head = v_cur->ne[0];
+     const int64_t n_head      = v_cur->ne[1];
+@@ -1992,38 +2309,109 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
+     GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]);
+ 
+     // opencoti F5 M2 headinfer — head-scatter into the GPU + host subsets.
+-    // Reached only when !v_trans and n_stream==1 (the split is gated off
+-    // otherwise at construction), so this mirrors the cpy_k store exactly.
+-    if (layer.gpu_heads > 0 && layer.v_cpu) {
++    // Reached only when !v_trans (the split is gated off at construction
++    // otherwise).
++    //
++    // opencoti F4 M3 Phase 4-D: per-stream M2 V scatter — same shape as
++    // cpy_k's M2 branch. Unified mode collapses to 2 set_rows on stream 0,
++    // byte-identical to pre-Phase-4.
++    if (layer.gpu_heads > 0
++        && !layer.v_cpu_per_stream.empty()
++        && layer.v_cpu_per_stream[0]) {
+         const int64_t g = layer.gpu_heads;
+         const int64_t c = n_head - g;
+-        ggml_tensor * vc = layer.v_cpu;
+-        ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur, n_embd_head*g, n_tokens, v_cur->nb[2], 0);
+-        ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur, n_embd_head*c, n_tokens, v_cur->nb[2], (size_t) (g*v_cur->nb[1]));
+-        ggml_tensor * store_g = ggml_set_rows(ctx, v,  cur_g, v_idxs);
+-        ggml_tensor * store_c = ggml_set_rows(ctx, vc, cur_c, v_idxs);
+-        return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0));
++        const uint32_t batch_ns = sinfo.n_stream();
++        GGML_ASSERT(batch_ns >= 1);
++
++        const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns);
++        GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens);
++
++        ggml_tensor * tie = nullptr;
++        for (uint32_t s = 0; s < batch_ns; ++s) {
++            ggml_tensor * v_s  = layer.v_per_stream[sinfo.strm[s]];
++            ggml_tensor * vc_s = layer.v_cpu_per_stream[sinfo.strm[s]];
++
++            const size_t per_stream_off = (size_t) s * tokens_per_stream * v_cur->nb[2];
++            ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur,
++                    n_embd_head*g, tokens_per_stream,
++                    v_cur->nb[2], per_stream_off);
++            ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur,
++                    n_embd_head*c, tokens_per_stream,
++                    v_cur->nb[2], per_stream_off + (size_t)(g*v_cur->nb[1]));
++            ggml_tensor * idxs_s = (batch_ns == 1)
++                ? v_idxs
++                : ggml_view_1d(ctx, v_idxs, tokens_per_stream,
++                        (size_t) s * tokens_per_stream * v_idxs->nb[0]);
++            ggml_tensor * store_g = ggml_set_rows(ctx, v_s,  cur_g, idxs_s);
++            ggml_tensor * store_c = ggml_set_rows(ctx, vc_s, cur_c, idxs_s);
++            ggml_tensor * pair = ggml_add(ctx,
++                    ggml_view_1d(ctx, store_g, 1, 0),
++                    ggml_view_1d(ctx, store_c, 1, 0));
++            if (!tie) {
++                tie = pair;
++            } else {
++                tie = ggml_add(ctx,
++                        ggml_view_1d(ctx, tie, 1, 0),
++                        ggml_view_1d(ctx, pair, 1, 0));
++            }
++        }
++        return tie;
+     }
+ 
+-    const int64_t n_stream = v->ne[2];
++    // opencoti F4 M3 Phase 4-C: per-stream V scatter. Two sub-branches
++    // depending on v_trans (FA vs non-FA layout).
++    //
++    // !v_trans (FA on): same shape as cpy_k — N separate ggml_set_rows on
++    // the 2-D-merged v_cur, sliced per stream. set_input_v_idxs emits
++    // LOCAL per-stream indices in non-unified mode. Unified mode (batch_ns
++    // == 1) keeps the simple one-set_rows fast path, byte-identical.
++    //
++    // v_trans (FA off, V transposed): each row is a single head element.
++    // The unified-mode path is preserved; multi-stream + v_trans is not a
++    // current workload (the v_trans layout requires no FA, while parallel
++    // multi-stream batches are always paired with FA). Asserted out as a
++    // safety net; future Phase 4-D/E may lift if a workload appears.
++    const uint32_t batch_ns = sinfo.n_stream();
++    GGML_ASSERT(batch_ns >= 1);
+ 
+-    // take this branch when FA is enabled (the V cache is not transposed)
+     if (!v_trans) {
+         v_cur = ggml_view_2d(ctx, v_cur, n_embd_gqa, n_tokens, v_cur->nb[2], 0);
+ 
+-        if (n_stream > 1) {
+-            const int64_t kv_size = get_size();
+-
+-            assert(n_embd_gqa == v->ne[0]);
+-            assert(kv_size    == v->ne[1]);
+-
+-            // merge the buffer across all streams because the idxs are global
+-            v = ggml_reshape_2d(ctx, v, n_embd_gqa, kv_size*n_stream);
++        if (batch_ns == 1) {
++            ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[0]];
++            return ggml_set_rows(ctx, v_s, v_cur, v_idxs);
++        }
++
++        const int64_t tokens_per_stream = n_tokens / (int64_t) batch_ns;
++        GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens);
++
++        ggml_tensor * tie = nullptr;
++        for (uint32_t s = 0; s < batch_ns; ++s) {
++            ggml_tensor * v_s   = layer.v_per_stream[sinfo.strm[s]];
++            ggml_tensor * cur_s = ggml_view_2d(ctx, v_cur,
++                    n_embd_gqa, tokens_per_stream,
++                    v_cur->nb[1],
++                    (size_t) s * tokens_per_stream * v_cur->nb[1]);
++            ggml_tensor * idxs_s = ggml_view_1d(ctx, v_idxs,
++                    tokens_per_stream,
++                    (size_t) s * tokens_per_stream * v_idxs->nb[0]);
++            ggml_tensor * store_s = ggml_set_rows(ctx, v_s, cur_s, idxs_s);
++            if (!tie) {
++                tie = store_s;
++            } else {
++                tie = ggml_add(ctx,
++                        ggml_view_1d(ctx, tie, 1, 0),
++                        ggml_view_1d(ctx, store_s, 1, 0));
++            }
+         }
+-
+-        return ggml_set_rows(ctx, v, v_cur, v_idxs);
++        return tie;
+     }
+ 
++    // v_trans path: not a multi-stream workload; preserve unified-mode
++    // behavior exactly and assert multi-stream out.
++    GGML_ASSERT(batch_ns == 1 && "v_trans + multi-stream is unsupported; v_trans implies !FA which is mutually exclusive with parallel batches");
++    ggml_tensor * v = layer.v_per_stream[sinfo.strm[0]];
++
+     if (ggml_row_size(v_cur->type, n_embd_gqa) == v_cur->nb[2]) {
+         // we can merge dims 0, 1 and 2
+         v_cur = ggml_reshape_2d(ctx, v_cur, n_embd_gqa, n_tokens);
+@@ -2119,11 +2507,23 @@ void llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ub
+     GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
+     int64_t * data = (int64_t *) dst->data;
+ 
+-    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
+-        const int64_t offs = sinfo.strm[s]*get_size();
++    // opencoti F4 M3 Phase 4-C/F: emit LOCAL per-stream indices unconditionally.
++    // Phase 4 ALWAYS scatters to per-stream tensors (cpy_k routes to
++    // k_per_stream[sinfo.strm[s]] which is sized kv_size, not kv_size*n_stream),
++    // so the index must be local to that stream's tensor. In unified mode
++    // (n_stream == 1, sinfo.strm[0] == 0) pre-Phase-4 offs was 0 already,
++    // so this is byte-identical.
++    //
++    // Phase 4-F bug fix: the previous version conditioned the offset
++    // on sinfo.n_stream() (per-batch streams), which left it global
++    // when a --parallel N server served a single-session batch
++    // (sinfo.n_stream() == 1 but sinfo.strm[0] != 0). The resulting
++    // global indices wrote out-of-bounds against the per-stream
++    // tensor and crashed CUDA decode.
+ 
++    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
+         for (uint32_t i = 0; i < sinfo.size(); ++i) {
+-            data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i];
++            data[s*sinfo.size() + i] = sinfo.idxs[s][i];
+         }
+     }
+ }
+@@ -2135,12 +2535,15 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub
+     GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
+     int64_t * data = (int64_t *) dst->data;
+ 
++    // opencoti F4 M3 Phase 4-C/F: same LOCAL-only fix as set_input_k_idxs.
++    // The v_trans branch's offset is also dropped — v_trans + multi-stream
++    // is asserted out in cpy_v, but the unified-mode v_trans path here is
++    // byte-identical regardless (sinfo.strm[0] == 0 makes offs == 0).
++
+     if (!v_trans) {
+         for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
+-            const int64_t offs = sinfo.strm[s]*get_size();
+-
+             for (uint32_t i = 0; i < sinfo.size(); ++i) {
+-                data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i];
++                data[s*sinfo.size() + i] = sinfo.idxs[s][i];
+             }
+         }
+     } else {
+@@ -2150,11 +2553,9 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub
+         const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa_max();
+ 
+         for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
+-            const int64_t offs = sinfo.strm[s]*kv_size*n_embd_v_gqa;
+-
+             for (uint32_t i = 0; i < sinfo.size(); ++i) {
+                 for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
+-                    data[s*sinfo.size()*n_embd_v_gqa + i*n_embd_v_gqa + j] = offs + j*kv_size + sinfo.idxs[s][i];
++                    data[s*sinfo.size()*n_embd_v_gqa + i*n_embd_v_gqa + j] = j*kv_size + sinfo.idxs[s][i];
+                 }
+             }
+         }
+@@ -2459,10 +2860,16 @@ size_t llama_kv_cache::total_size() const {
+ }
+ 
+ size_t llama_kv_cache::size_k_bytes() const {
++    // opencoti F4 M3 Phase 4: sum bytes across all per-stream tensors of
++    // each layer. In unified mode (n_stream == 1) this is byte-identical
++    // to ggml_nbytes(layer.k) of pre-Phase-4. M2's host subset is
++    // accounted separately (it lives in a different buffer-type).
+     size_t size_k_bytes = 0;
+ 
+     for (const auto & layer : layers) {
+-        size_k_bytes += ggml_nbytes(layer.k);
++        for (auto * k : layer.k_per_stream) {
++            if (k) size_k_bytes += ggml_nbytes(k);
++        }
+     }
+ 
+     return size_k_bytes;
+@@ -2472,7 +2879,9 @@ size_t llama_kv_cache::size_v_bytes() const {
+     size_t size_v_bytes = 0;
+ 
+     for (const auto & layer : layers) {
+-        size_v_bytes += layer.v ? ggml_nbytes(layer.v) : 0;
++        for (auto * v : layer.v_per_stream) {
++            if (v) size_v_bytes += ggml_nbytes(v);
++        }
+     }
+ 
+     return size_v_bytes;
+@@ -2570,6 +2979,19 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co
+ 
+     const auto & cparams = lctx->get_cparams();
+ 
++    // opencoti F4 M3 Phase 4-C: k-shift loops over per-stream tensors.
++    // With per-stream owning tensors, each stream's K is a separate buffer,
++    // so we build N rope-shift ops per layer (one per stream). The k_shift
++    // input tensor has length get_size()*n_stream, with stream s occupying
++    // indices [s*get_size(), (s+1)*get_size()) — set_input_k_shift writes
++    // them in that order. Each per-stream rope-shift consumes its
++    // contiguous slice via ggml_view_1d on k_shift.
++    //
++    // In unified mode (n_stream == 1), the loop runs once with stream 0,
++    // and the k_shift view degenerates to the full tensor (offset 0,
++    // length get_size()). Byte-identical to pre-Phase-4 — same rope-shift
++    // op shape and inputs.
++
+     for (const auto & layer : layers) {
+         const uint32_t il = layer.il;
+ 
+@@ -2585,16 +3007,24 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co
+ 
+         ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
+ 
+-        ggml_tensor * k =
+-            ggml_view_3d(ctx, layer.k,
+-                n_rot, n_head_kv, get_size()*n_stream,
+-                ggml_row_size(layer.k->type, n_embd_head_k),
+-                ggml_row_size(layer.k->type, n_embd_k_gqa),
+-                ggml_row_size(layer.k->type, n_embd_nope));
++        for (uint32_t s = 0; s < n_stream; ++s) {
++            ggml_tensor * layer_k = layer.k_per_stream[s];
++            ggml_tensor * k =
++                ggml_view_3d(ctx, layer_k,
++                    n_rot, n_head_kv, get_size(),
++                    ggml_row_size(layer_k->type, n_embd_head_k),
++                    ggml_row_size(layer_k->type, n_embd_k_gqa),
++                    ggml_row_size(layer_k->type, n_embd_nope));
++
++            ggml_tensor * k_shift_s = (n_stream == 1)
++                ? inp->k_shift
++                : ggml_view_1d(ctx, inp->k_shift, get_size(),
++                        (size_t) s * get_size() * inp->k_shift->nb[0]);
+ 
+-        ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il);
++            ggml_tensor * cur = build_rope_shift(cparams, ctx, k, k_shift_s, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il);
+ 
+-        ggml_build_forward_expand(gf, cur);
++            ggml_build_forward_expand(gf, cur);
++        }
+     }
+ 
+     res->add_input(std::move(inp));
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -286,19 +286,57 @@ private:
+         // note: can be different from the layer index in the KV cache
+         uint32_t il;
+ 
+-        ggml_tensor * k;
+-        ggml_tensor * v;
+-
++        // opencoti F4 M3 Phase 4 — per-stream tensor split.
++        // Each element is an OWNING 3-D tensor of shape
++        // [n_embd_k_gqa, kv_size, 1] (dim 3 == 1 keeps stride math
++        // identical to the pre-Phase-4 single-tensor layout). Vector
++        // size always equals n_stream.
++        //
++        // Unified mode (n_stream == 1): k_per_stream[0] is byte-
++        // identical to the pre-Phase-4 `k` field (same buft, same
++        // ctx, same offsets). Non-unified mode (n_stream > 1): each
++        // stream's tensor lives in its own (possibly per-stream)
++        // backend buffer (Phase 4-B); cross-stream operations are
++        // buffer-to-buffer copies.
++        std::vector<ggml_tensor *> k_per_stream;
++        std::vector<ggml_tensor *> v_per_stream;
++
++        // k_stream[s] / v_stream[s] are 2-D views into
++        // k_per_stream[s] / v_per_stream[s]. Kept for backward compat
++        // with consumers that index by stream; same shape and offset
++        // semantics as before. With per-stream tensors the view's
++        // offset within its owning tensor is always 0 (each stream
++        // owns the whole 3-D extent of its tensor with dim 3 == 1).
+         std::vector<ggml_tensor *> k_stream;
+         std::vector<ggml_tensor *> v_stream;
+ 
+-        // opencoti F5 M2 headinfer — head-residency split. When gpu_heads > 0,
+-        // `k`/`v` hold the first gpu_heads KV heads on the layer's (GPU) buffer
+-        // and `k_cpu`/`v_cpu` hold the remaining (n_head_kv - gpu_heads) heads
+-        // on a host buffer. gpu_heads == 0 means no split (k/v hold all heads).
+-        ggml_tensor * k_cpu = nullptr;
+-        ggml_tensor * v_cpu = nullptr;
++        // opencoti F5 M2 headinfer — per-stream now. When gpu_heads > 0,
++        // k_per_stream[s] / v_per_stream[s] hold the first gpu_heads
++        // KV heads on the layer's device buffer, and
++        // k_cpu_per_stream[s] / v_cpu_per_stream[s] hold the remaining
++        // (n_head_kv - gpu_heads) heads on the layer's host buffer.
++        // Vectors are empty when no split (gpu_heads == 0).
++        std::vector<ggml_tensor *> k_cpu_per_stream;
++        std::vector<ggml_tensor *> v_cpu_per_stream;
+         uint32_t gpu_heads = 0;
++
++        // Convenience accessors for unified-mode (n_stream == 1) call
++        // sites. Returns nullptr in non-unified mode — that's the
++        // signal that the caller must use per-stream access instead.
++        // Phase 4-A uses these at sites that haven't yet been migrated
++        // to per-stream awareness; Phase 4-C/D lift them.
++        inline ggml_tensor * k_unified() const {
++            return k_per_stream.size() == 1 ? k_per_stream[0] : nullptr;
++        }
++        inline ggml_tensor * v_unified() const {
++            return v_per_stream.size() == 1 ? v_per_stream[0] : nullptr;
++        }
++        inline ggml_tensor * k_cpu_unified() const {
++            return k_cpu_per_stream.size() == 1 ? k_cpu_per_stream[0] : nullptr;
++        }
++        inline ggml_tensor * v_cpu_unified() const {
++            return v_cpu_per_stream.size() == 1 ? v_cpu_per_stream[0] : nullptr;
++        }
+     };
+ 
+     bool v_trans = true;  // the value tensor is transposed

patches/0032-m2-state-io.patch

@@ -0,0 +1,205 @@
+From: opencoti
+Subject: F5 M2 — M2-aware state I/O (bug-222 fix)
+
+Bench: perf/llamafile/advanced-kv-stack.bench.ts (Gemma 4 A4B + 31B), bug-222 fix
+Milestone: F5 M2 hardening — see docs/features/advanced_kv.md, .wolf/buglog.json bug-222
+
+Pre-existing M2 head split bug surfaced on Gemma 4 iswa: state_write_data /
+state_read_data read/write the K and V cache assuming a single tensor per
+layer with full n_embd_k_gqa / n_embd_v_gqa row size. With M2 head split
+active the layer's tensor (k_stream[s] / v_stream[s]) holds ONLY the GPU
+half (n_embd_k_gpu width), and the CPU half lives in a separate
+k_cpu_per_stream[s] / v_cpu_per_stream[s] tensor. Reading buf_size =
+range_size * row_full from a tensor that has range_size * row_gpu bytes
+crashes with "tensor read out of bounds" in ggml_backend_tensor_get
+(ggml-backend.cpp:314) as soon as the per-range write exceeds the GPU
+half's nbytes — reproducible on Gemma 4 A4B & 31B at frac=0.5 with any
+prompt that crosses a 512-token microbatch boundary and triggers a
+context checkpoint (the cache_prompt server feature snapshots ≥ 634
+cells after the second prompt batch).
+
+Fix: detect M2 split per layer (layer.gpu_heads > 0 &&
+layer.k_cpu_per_stream[s]), emit GPU half (row_gpu) then CPU half
+(row_cpu) per range. Total bytes per range stay at range_size * row_full
+(header advertises the full row size unchanged) so io stream accounting
+is preserved; on-disk byte order within a range differs from upstream
+(GPU-first|CPU-first vs interleaved) but writer/reader are symmetric so
+round-trips are consistent. M2 inactive (no head_cpu) path is taken
+byte-identical to upstream — preserves Qwen / unified-mode behavior.
+V branch only fires in !v_trans (M2 is gated off when v_trans).
+
+Validated:
+  - Gemma 4 A4B-98e-v6-coder-it-Q4_K_M: M5 stack bench 8/9 PASS
+    (C5 byte-equality A2/A3 PASS, C9 perf threshold known cost).
+  - Gemma 4 31B-it-Q4_K_M: M5 stack bench 8/9 PASS, same shape.
+  - Qwen 2.5 1.5B Instruct: regression shield — M5 8/9, M2 6/6, M3 4/4,
+    M0 5/5 all PASS unchanged.
+
+Bug-222 reproducer: /tmp/gemma-crash-probe.py (cycles reps 10..50 of the
+bench prompt against an A2 server; pre-fix crashes at reps=30, post-fix
+clean through reps=35).
+
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -3190,11 +3190,43 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t
+         const uint64_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa);
+         io.write(&k_size_row, sizeof(k_size_row));
+ 
+-        // Read each range of cells of k_size length and write out
+-        for (const auto & range : cr.data) {
+-            const size_t range_size = range.second - range.first;
+-            const size_t buf_size = range_size * k_size_row;
+-            io.write_tensor(k, range.first * k_size_row, buf_size);
++        // opencoti bug-222 fix — F5 M2 split-aware state write.
++        // When M2 head split is active, `k` (k_stream[s]) holds only the
++        // GPU half of the layer's heads; the CPU half lives in
++        // k_cpu_per_stream[s]. The pre-fix path read `range_size * k_size_row`
++        // bytes from a tensor that only has `range_size * row_gpu` bytes →
++        // out-of-bounds crash on any range_size that pushes buf_size past
++        // the per-stream GPU tensor's nbytes. Fix: detect M2 split, emit
++        // GPU half then CPU half per range. Total bytes per range equals
++        // `range_size * k_size_row` (the header advertises this), so on-disk
++        // accounting matches; the layout differs only in within-range order
++        // (GPU-first|CPU-first vs interleaved). Reader (state_read_data) is
++        // symmetric so a save/load round-trip is consistent. When M2 is OFF
++        // (no head_cpu) the original path is taken byte-identical.
++        const bool m2_split_active = layer.gpu_heads > 0
++                                  && !layer.k_cpu_per_stream.empty()
++                                  && layer.k_cpu_per_stream[cr.strm];
++        if (m2_split_active) {
++            auto * k_cpu = layer.k_cpu_per_stream[cr.strm];
++            const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++            const uint32_t n_head_kv     = hparams.n_head_kv(il);
++            const uint32_t n_embd_k_gpu  = layer.gpu_heads * n_embd_head_k;
++            const uint32_t n_embd_k_cpu  = (n_head_kv - layer.gpu_heads) * n_embd_head_k;
++            const uint64_t k_row_gpu     = ggml_row_size(k->type,     n_embd_k_gpu);
++            const uint64_t k_row_cpu     = ggml_row_size(k_cpu->type, n_embd_k_cpu);
++            GGML_ASSERT(k_row_gpu + k_row_cpu == k_size_row);
++            for (const auto & range : cr.data) {
++                const size_t range_size = range.second - range.first;
++                io.write_tensor(k,     range.first * k_row_gpu, range_size * k_row_gpu);
++                io.write_tensor(k_cpu, range.first * k_row_cpu, range_size * k_row_cpu);
++            }
++        } else {
++            // Read each range of cells of k_size length and write out
++            for (const auto & range : cr.data) {
++                const size_t range_size = range.second - range.first;
++                const size_t buf_size = range_size * k_size_row;
++                io.write_tensor(k, range.first * k_size_row, buf_size);
++            }
+         }
+     }
+ 
+@@ -3217,11 +3249,33 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t
+             const uint64_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa);
+             io.write(&v_size_row, sizeof(v_size_row));
+ 
+-            // Read each range of cells of v_size length and write out
+-            for (const auto & range : cr.data) {
+-                const size_t range_size = range.second - range.first;
+-                const size_t buf_size = range_size * v_size_row;
+-                io.write_tensor(v, range.first * v_size_row, buf_size);
++            // opencoti bug-222 fix — F5 M2 split-aware V state write
++            // (same pattern as K above). M2 is gated off when v_trans, so
++            // this branch is reached only with the non-transposed V layout.
++            const bool m2_split_active = layer.gpu_heads > 0
++                                      && !layer.v_cpu_per_stream.empty()
++                                      && layer.v_cpu_per_stream[cr.strm];
++            if (m2_split_active) {
++                auto * v_cpu = layer.v_cpu_per_stream[cr.strm];
++                const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++                const uint32_t n_head_kv     = hparams.n_head_kv(il);
++                const uint32_t n_embd_v_gpu  = layer.gpu_heads * n_embd_head_v;
++                const uint32_t n_embd_v_cpu  = (n_head_kv - layer.gpu_heads) * n_embd_head_v;
++                const uint64_t v_row_gpu     = ggml_row_size(v->type,     n_embd_v_gpu);
++                const uint64_t v_row_cpu     = ggml_row_size(v_cpu->type, n_embd_v_cpu);
++                GGML_ASSERT(v_row_gpu + v_row_cpu == v_size_row);
++                for (const auto & range : cr.data) {
++                    const size_t range_size = range.second - range.first;
++                    io.write_tensor(v,     range.first * v_row_gpu, range_size * v_row_gpu);
++                    io.write_tensor(v_cpu, range.first * v_row_cpu, range_size * v_row_cpu);
++                }
++            } else {
++                // Read each range of cells of v_size length and write out
++                for (const auto & range : cr.data) {
++                    const size_t range_size = range.second - range.first;
++                    const size_t buf_size = range_size * v_size_row;
++                    io.write_tensor(v, range.first * v_size_row, buf_size);
++                }
+             }
+         }
+     } else {
+@@ -3433,7 +3487,36 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
+         }
+ 
+         if (cell_count) {
+-            if (sinfo.is_contiguous()) {
++            // opencoti bug-222 fix — F5 M2 split-aware state read (K).
++            // Symmetric to state_write_data: when M2 head split is active,
++            // read the GPU half then the CPU half and dispatch to the
++            // respective per-stream tensors. Total bytes equal
++            // `cell_count * k_size_row` (the header advertises this) so the
++            // sequential io stream matches what state_write_data wrote.
++            const bool m2_split_active = layer.gpu_heads > 0
++                                      && !layer.k_cpu_per_stream.empty()
++                                      && layer.k_cpu_per_stream[strm];
++            if (m2_split_active) {
++                auto * k_cpu = layer.k_cpu_per_stream[strm];
++                const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
++                const uint32_t n_head_kv     = hparams.n_head_kv(il);
++                const uint32_t n_embd_k_gpu  = layer.gpu_heads * n_embd_head_k;
++                const uint32_t n_embd_k_cpu  = (n_head_kv - layer.gpu_heads) * n_embd_head_k;
++                const uint64_t k_row_gpu     = ggml_row_size(k->type,     n_embd_k_gpu);
++                const uint64_t k_row_cpu     = ggml_row_size(k_cpu->type, n_embd_k_cpu);
++                GGML_ASSERT(k_row_gpu + k_row_cpu == k_size_row);
++                if (sinfo.is_contiguous()) {
++                    io.read_tensor(k,     sinfo.head() * k_row_gpu, cell_count * k_row_gpu);
++                    io.read_tensor(k_cpu, sinfo.head() * k_row_cpu, cell_count * k_row_cpu);
++                } else {
++                    for (uint32_t i = 0; i < cell_count; ++i) {
++                        io.read_tensor(k, sinfo.idxs[0][i] * k_row_gpu, k_row_gpu);
++                    }
++                    for (uint32_t i = 0; i < cell_count; ++i) {
++                        io.read_tensor(k_cpu, sinfo.idxs[0][i] * k_row_cpu, k_row_cpu);
++                    }
++                }
++            } else if (sinfo.is_contiguous()) {
+                 // Fast path: contiguous cells, single memcpy
+                 io.read_tensor(k, sinfo.head() * k_size_row, cell_count * k_size_row);
+             } else {
+@@ -3476,7 +3559,33 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
+             }
+ 
+             if (cell_count) {
+-                if (sinfo.is_contiguous()) {
++                // opencoti bug-222 fix — F5 M2 split-aware state read (V),
++                // same pattern as K. M2 is gated off when v_trans, so this
++                // branch is only reached with the non-transposed V layout.
++                const bool m2_split_active = layer.gpu_heads > 0
++                                          && !layer.v_cpu_per_stream.empty()
++                                          && layer.v_cpu_per_stream[strm];
++                if (m2_split_active) {
++                    auto * v_cpu = layer.v_cpu_per_stream[strm];
++                    const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
++                    const uint32_t n_head_kv     = hparams.n_head_kv(il);
++                    const uint32_t n_embd_v_gpu  = layer.gpu_heads * n_embd_head_v;
++                    const uint32_t n_embd_v_cpu  = (n_head_kv - layer.gpu_heads) * n_embd_head_v;
++                    const uint64_t v_row_gpu     = ggml_row_size(v->type,     n_embd_v_gpu);
++                    const uint64_t v_row_cpu     = ggml_row_size(v_cpu->type, n_embd_v_cpu);
++                    GGML_ASSERT(v_row_gpu + v_row_cpu == v_size_row);
++                    if (sinfo.is_contiguous()) {
++                        io.read_tensor(v,     sinfo.head() * v_row_gpu, cell_count * v_row_gpu);
++                        io.read_tensor(v_cpu, sinfo.head() * v_row_cpu, cell_count * v_row_cpu);
++                    } else {
++                        for (uint32_t i = 0; i < cell_count; ++i) {
++                            io.read_tensor(v, sinfo.idxs[0][i] * v_row_gpu, v_row_gpu);
++                        }
++                        for (uint32_t i = 0; i < cell_count; ++i) {
++                            io.read_tensor(v_cpu, sinfo.idxs[0][i] * v_row_cpu, v_row_cpu);
++                        }
++                    }
++                } else if (sinfo.is_contiguous()) {
+                     // Fast path: contiguous cells, single memcpy
+                     io.read_tensor(v, sinfo.head() * v_size_row, cell_count * v_size_row);
+                 } else {

patches/0033-cpy-tie-blck-align.patch

@@ -0,0 +1,100 @@
+opencoti F5 bug-225 — block-align cpy_k/cpy_v dependency-tie views over quantized KV
+
+At 256k + q8_0 KV with M2 head split (--headinfer-gpu-heads-frac < 1.0) on Gemma 4
+iswa, server boot crashed during common_init_result's graph_reserve with:
+  ggml.c:1301: assert(ne % ggml_blck_size(type) == 0) failed
+
+Root cause: the M2 and Phase-4 branches of cpy_k / cpy_v built a graph-topology
+dependency tie via `ggml_view_1d(ctx, store, 1, 0)` over the K/V cache scatter
+result.  The tie value is never read by computation — it only forces multiple
+per-stream / per-head writes to be reachable from one ggml_build_forward_expand
+call.  At f16 (block size 1), ne=1 passes ggml_row_size silently.  At q8_0
+(block size 32), ne=1 fails the assert at graph_reserve before any inference
+runs.
+
+Fix at all 4 callsites (cpy_k M2 branch + Phase-4 multi-stream branch; cpy_v
+M2 branch + Phase-4 !v_trans multi-stream branch — 12 view sites total):
+  - Replace `ne = 1` with `ggml_blck_size(target->type)` so the view passes
+    the quantized-alignment check.
+  - Wrap each view in `ggml_cast(..., GGML_TYPE_F32)` so the subsequent
+    `ggml_add` survives CUDA bin_bcast's F32/F16-only check (binbcast.cu:376).
+
+At f16 KV both wraps are no-ops (blck_size==1, cast on f16-view is byte-identical
+for ne=1 tie semantics never read).  At q8_0 the wraps unblock 256k boot.
+
+Verification:
+  - Boot at 4096 / 32768 / 262144 + q8_0 + M2 frac=0.5 on Gemma 4 A4B Q4_K_M:
+    all succeed (no graph_reserve crash).
+  - RULER vt@4096 + vt@32768 + niah_single_1@{4096,32768} ours-c1 scored 100/100
+    against the post-fix binary on 2026-05-29 — confirms no semantic regression
+    at moderate ctx.
+
+Related: bug-222 (n_kv/view stride on iswa tail microbatch); bug-224 (concat-F32
+on iswa); bug-226 (256k semantic collapse — different root cause, separate fix).
+
+Patch order: applies after 0031-per-stream-split (Phase 4) and 0032-m2-state-io.
+Cosmocc-only edit; CUDA DSO unchanged.
+
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -2231,14 +2231,14 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
+             ggml_tensor * store_g = ggml_set_rows(ctx, k_s,  cur_g, idxs_s);
+             ggml_tensor * store_c = ggml_set_rows(ctx, kc_s, cur_c, idxs_s);
+             ggml_tensor * pair = ggml_add(ctx,
+-                    ggml_view_1d(ctx, store_g, 1, 0),
+-                    ggml_view_1d(ctx, store_c, 1, 0));
++                    ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32),
++                    ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32));
+             if (!tie) {
+                 tie = pair;
+             } else {
+                 tie = ggml_add(ctx,
+-                        ggml_view_1d(ctx, tie, 1, 0),
+-                        ggml_view_1d(ctx, pair, 1, 0));
++                        ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32),
++                        ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32));
+             }
+         }
+         return tie;
+@@ -2287,8 +2287,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
+             tie = store_s;
+         } else {
+             tie = ggml_add(ctx,
+-                    ggml_view_1d(ctx, tie, 1, 0),
+-                    ggml_view_1d(ctx, store_s, 1, 0));
++                    ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32),
++                    ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32));
+         }
+     }
+     return tie;
+@@ -2345,14 +2345,14 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
+             ggml_tensor * store_g = ggml_set_rows(ctx, v_s,  cur_g, idxs_s);
+             ggml_tensor * store_c = ggml_set_rows(ctx, vc_s, cur_c, idxs_s);
+             ggml_tensor * pair = ggml_add(ctx,
+-                    ggml_view_1d(ctx, store_g, 1, 0),
+-                    ggml_view_1d(ctx, store_c, 1, 0));
++                    ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32),
++                    ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32));
+             if (!tie) {
+                 tie = pair;
+             } else {
+                 tie = ggml_add(ctx,
+-                        ggml_view_1d(ctx, tie, 1, 0),
+-                        ggml_view_1d(ctx, pair, 1, 0));
++                        ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32),
++                        ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32));
+             }
+         }
+         return tie;
+@@ -2400,8 +2400,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
+                 tie = store_s;
+             } else {
+                 tie = ggml_add(ctx,
+-                        ggml_view_1d(ctx, tie, 1, 0),
+-                        ggml_view_1d(ctx, store_s, 1, 0));
++                        ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32),
++                        ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32));
+             }
+         }
+         return tie;

patches/0034-m2-pinned-host.patch

@@ -0,0 +1,80 @@
+opencoti F5 M7-A (0034) — pinned host buffer for M2 CPU-half KV
+opencoti note (2026-05-31, bug-250): hunk regenerated via snapshot-diff against the post-0033 bootstrap so it strict git-applies — the original removed two trailing-whitespace lines absent from the current tree. Content unchanged (pinned cpu_buft resolution); verified against the live r2a tree.
+
+
+Switches the M2 head-split CPU-half allocation from
+ggml_backend_cpu_buffer_type() (pageable) to ggml_backend_dev_host_buffer_type(dev)
+(CUDA pinned host memory) when the layer is offloaded to a CUDA device. Falls
+back to pageable CPU buft when no CUDA dev / no dev_host_buffer_type.
+
+Why: pageable host memory caps PCIe transfers at ~50% of theoretical
+bandwidth because every DMA has to first pin the page or stage through a
+driver-side bounce buffer. Pinned host memory unlocks full PCIe bandwidth
+on every M2 CPU→GPU stream-back AND is the foundation Rolling KV (F5 M7)
+builds on — without pinned residency, the M7 streaming pipeline can't hit
+its compute/copy-overlap budget at PCIe 3.0 x8.
+
+Mechanics:
+  - Layer scope (~line 270): resolve cpu_buft alongside the existing
+    layer GPU buft. Read dev_host_buffer_type(dev); fall back to plain
+    cpu_buffer_type if unsupported.
+  - Per-stream scope (~line 391): consume the layer-scope cpu_buft
+    instead of calling ggml_backend_cpu_buffer_type() inline.
+
+f16 and CPU-only paths byte-identical to pre-0034 (dev_host_buffer_type
+returns null → fallback fires → identical alloc). q8_0 + M2 + CUDA paths
+get pinned residency. Foundation for #296 (Rolling KV / M7).
+
+Verification:
+  - Cosmocc build clean (~80s incremental).
+  - Smoke at 32k + f16 + M2 frac=0.5 + M0/M1/M3 flags engaged on Gemma 4
+    A4B Q4_K_M: server boots clean, /v1/completions returns coherent
+    tokens (no crash, no DSO mismatch).
+  - Does NOT fix bug-226 (256k+q8_0 semantic collapse — confirmed
+    deterministic via 1-sample canary 2026-05-29 21:51, scored 0/1 with
+    same wrong output 'LJXLD' as the original --no-kv-unified failure;
+    pinning is orthogonal to the corruption mechanism).
+  - Held for bug-226 root cause before shipping as default — pinning a
+    broken stack doesn't fix it but doesn't make it worse either.
+
+Patch order: applies after 0033-cpy-tie-blck-align. Cosmocc-only edit;
+CUDA DSO unchanged (host-side buffer-type resolution, no kernel changes).
+
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -267,11 +267,20 @@ llama_kv_cache::llama_kv_cache(
+ 
+         ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type();
+ 
++        // opencoti F5 M7-A (0034) — resolved at layer scope alongside the
++        // per-layer GPU buft so the M2 CPU-half allocation below (line ~390)
++        // can reach pinned host memory when CUDA is offloaded. Falls back to
++        // pageable CPU buft when no CUDA dev / no dev_host_buffer_type. Pinned
++        // host buffers DMA at full PCIe bandwidth; pageable hits ~50%.
++        ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type();
+         if (offload) {
+             auto * dev = model.dev_layer(il);
+             buft = ggml_backend_dev_buffer_type(dev);
+ 
+             dev_name = ggml_backend_dev_name(dev);
++
++            ggml_backend_buffer_type_t pinned = ggml_backend_dev_host_buffer_type(dev);
++            if (pinned) cpu_buft = pinned;
+         }
+ 
+         LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name);
+@@ -374,7 +383,12 @@ llama_kv_cache::llama_kv_cache(
+             // when gpu_heads == 0 so headinfer_split_active() can test
+             // by vector size.
+             if (gpu_heads > 0) {
+-                ggml_context * ctx_cpu_s = ctx_for_buft(ggml_backend_cpu_buffer_type(), s);
++                // opencoti F5 M7-A (0034) — was ggml_backend_cpu_buffer_type()
++                // (pageable). Now uses the layer-scope cpu_buft resolved above:
++                // pinned host buffer when CUDA is the layer dev, plain CPU buft
++                // otherwise. Pinned unlocks full PCIe DMA for M2's CPU-half
++                // stream-back AND is the foundation Rolling KV (M7) builds on.
++                ggml_context * ctx_cpu_s = ctx_for_buft(cpu_buft, s);
+                 if (!ctx_cpu_s) {
+                     throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache");
+                 }

patches/0035-concat-q-block-aware.patch

@@ -0,0 +1,209 @@
+opencoti F5 bug-226 — block-aware ggml-cpu concat for quantized types
+
+At 32k+q8_0 KV with M2 head split (--headinfer-gpu-heads-frac < 1.0)
+without NEO orchestration, the M2 head-axis concat path in get_k/get_v
+(llama-kv-cache.cpp:1881-1910) produced semantically corrupt K/V that
+the downstream flash-attn read as garbage, scoring 0.00% on RULER vt
+with completely wrong output (e.g. predicting 'SHRUU' instead of the
+five expected variables). 5-cell bisection at 32k+q8_0 isolated M2 alone
+(--headinfer-gpu-heads-frac 0.5) as the failing config; M0, M1, M3, and
+vanilla scored 100.
+
+Root cause: ggml_concat on block-quantized tensors falls through
+multiple broken paths.
+
+The M2 non-NEO get_k/get_v branch returns:
+
+    ggml_concat(ctx, ggml_view_4d(k_s, head_dim, n_head_gpu, n_kv, 1),
+                     ggml_view_4d(kc_s, head_dim, n_head_cpu, n_kv, 1), 1)
+
+both views are q8_0. The CUDA backend rejects this (concat.cu:165-167
+asserts F32-only — task #253's earlier supports_op tightening was
+correct), so the scheduler dispatches to CPU. CPU concat falls into
+ggml_compute_forward_concat_any (ops.cpp:1894-1935) for any non-
+F16/BF16/I16/I8/F32/I32 type. concat_any iterates i0 over LOGICAL
+elements (0..ne0) with stride nb00, calling memcpy(len = type_size).
+For q8_0, nb00 = type_size = 36 bytes (one block of 32 elements), but
+the code treats it as a per-element stride — reading blocks at offsets
+0, 36, 72, ..., 127·36 = 4572 bytes from each row start, well past
+the actual row of (ne0/32)·36 = 144 bytes. Result: q8_0 blocks
+scrambled into uninterpretable data → FA reads garbage K → output is
+task-format-correct but value-wrong.
+
+At f16 KV (block size 1) this latent bug never triggered because
+concat dispatches to ggml_compute_forward_concat_f16 which correctly
+handles fp16 per-element. The full F5 stack at f16 + 32k scored 100
+in the morning RULER comparison, masking the q8_0 case.
+
+Fix: a new ggml_compute_forward_concat_q that copies entire dim-0 rows
+at once for block-quantized types. Dispatched from
+ggml_compute_forward_concat's default branch when ggml_blck_size > 1.
+
+For dim != 0 (the M2 head-axis case): each (i1, i2, i3) maps to either
+src0 or src1 (mutually exclusive); one memcpy of (ne0/blck)·type_size
+bytes per row. For dim == 0: dst's dim-0 row = src0_row || src1_row,
+asserted block-aligned at the split; two memcpys per row.
+
+Non-block-quantized (F32/F16/BF16/I8/I16/I32) paths unchanged. The
+ggml-cuda CUDA path stays F32-only — that's correct since this fix
+lands at the CPU fallback the scheduler already routes to.
+
+Lives in ggml-cpu only — no CUDA DSO rebuild needed. Compatible with
+incremental cosmocc rebuild via `OPENCOTI_NO_CCACHE=1 bun run
+build:llamafile:make`.
+
+Verification:
+- T2 config smoke (M2-only, no NEO, q8_0, 32k, vt n=1) — bug repro
+  pre-fix scored 0.00% with pred='SHRUU'; post-fix expected 100.00%.
+- Full F5 stack (M0+M1+M2+M3+NEO+Phase4) at q8_0 + 32k — unchanged
+  from morning baseline (100%).
+- All M0/M1/M2/M3/M5 unified-mode regression-shield benches — unchanged
+  (cosine ≥ 0.999); concat_q only fires on block-quantized types, and
+  f16/bf16/f32 KV paths route to their own concat_fN as before.
+
+apply order: after 0034-m2-pinned-host (this is a ggml-cpu fix; the
+two are independent so order is bookkeeping only).
+
+Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
+
+diff --git a/llama.cpp/ggml/src/ggml-cpu/ops.cpp b/llama.cpp/ggml/src/ggml-cpu/ops.cpp
+--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp
++++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp
+@@ -2078,6 +2078,119 @@ static void ggml_compute_forward_concat_f32(
+     }
+ }
+ 
++// opencoti F5 bug-226 fix (0035) — block-quantized concat.
++//
++// concat_any (above) does per-logical-element memcpy(len = type_size),
++// stepping i0 from 0..ne0 with stride nb00. For block-quantized types
++// (q8_0, q4_0, …) the parent tensor stores ne0 LOGICAL elements packed
++// in ne0/blck blocks of type_size bytes each, with nb00 = type_size
++// (one block, not one element). concat_any's loop therefore reads
++// blocks at offsets 0, type_size, 2·type_size, … up to ne0·type_size —
++// well past the actual row of (ne0/blck)·type_size bytes. The result
++// is scrambled block data, which the downstream FA reads as garbage K
++// (root cause of bug-226 at M2 head-axis q8_0 K/V concat).
++//
++// Fix: for block-quantized types, copy entire dim-0 rows at once.
++// Each dim-0 row of src0/src1/dst is contiguous (nb00 = type_size,
++// ne0/blck blocks per row, no gaps), so a single memcpy per
++// (i1, i2, i3) selection on each source captures the right bytes.
++//
++// - dim != 0 (M2 head-axis case): per (i1, i2, i3), the whole dim-0
++//   row comes from src0 or src1 (mutually exclusive). One memcpy.
++// - dim == 0 (split on dim 0): dst's dim-0 row = src0's dim-0 row ||
++//   src1's dim-0 row, concatenated head-to-tail. The split MUST land
++//   on a block boundary (ne00 % blck == 0); when it does, two memcpys.
++//
++// The M2 split in llama-kv-cache.cpp:get_k/get_v concats on dim 1
++// (head axis), so the dim != 0 branch is the hot path. dim 0 is
++// covered for completeness — any future quantized-feature-axis split
++// pattern lands correctly without re-discovering this bug.
++static void ggml_compute_forward_concat_q(
++    const ggml_compute_params * params,
++    ggml_tensor * dst) {
++
++    const ggml_tensor * src0 = dst->src[0];
++    const ggml_tensor * src1 = dst->src[1];
++
++    const size_t  type_size = ggml_type_size(src0->type);
++    const int64_t blck      = ggml_blck_size(src0->type);
++
++    GGML_ASSERT(blck > 1 && "concat_q is for block-quantized types");
++    GGML_ASSERT(ggml_type_size(src1->type) == type_size);
++    GGML_ASSERT(ggml_blck_size(src1->type) == blck);
++    GGML_ASSERT(ggml_type_size(dst->type)  == type_size);
++    GGML_ASSERT(ggml_blck_size(dst->type)  == blck);
++
++    const int ith = params->ith;
++    const int nth = params->nth;
++
++    GGML_TENSOR_BINARY_OP_LOCALS
++
++    const int32_t dim = ggml_get_op_params_i32(dst, 0);
++
++    GGML_ASSERT(dim >= 0 && dim < 4);
++
++    // Both srcs must have block-aligned dim-0 sizes — guaranteed by ggml
++    // for any well-formed quantized tensor. We assert anyway because if
++    // a caller violates this, the row math below silently corrupts data.
++    GGML_ASSERT((ne00 % blck) == 0);
++    GGML_ASSERT((ne10 % blck) == 0);
++
++    // Bytes per dim-0 row of each source / dst.
++    const size_t row_bytes_src0 = (ne00 / blck) * type_size;
++    const size_t row_bytes_src1 = (ne10 / blck) * type_size;
++    const size_t row_bytes_dst  = (ne0  / blck) * type_size;
++
++    if (dim == 0) {
++        // dst row = src0 row (head) || src1 row (tail). Both fill at the
++        // block boundary. For non-concat dims i1/i2/i3 must match between
++        // src0 and src1 (ggml concat invariant), so no per-dim mux logic.
++        GGML_ASSERT(row_bytes_src0 + row_bytes_src1 == row_bytes_dst);
++        for (int i3 = 0; i3 < ne3; i3++) {
++            for (int i2 = ith; i2 < ne2; i2 += nth) {
++                for (int i1 = 0; i1 < ne1; i1++) {
++                    const char * x0 = (const char *)src0->data
++                                    + i1*nb01 + i2*nb02 + i3*nb03;
++                    const char * x1 = (const char *)src1->data
++                                    + i1*nb11 + i2*nb12 + i3*nb13;
++                    char * y = (char *)dst->data
++                             + i1*nb1 + i2*nb2 + i3*nb3;
++                    memcpy(y,                   x0, row_bytes_src0);
++                    memcpy(y + row_bytes_src0,  x1, row_bytes_src1);
++                }
++            }
++        }
++        return;
++    }
++
++    // dim != 0: each (i1, i2, i3) maps to either src0 or src1, never both.
++    // row_bytes is shared (src0/src1/dst all have the same dim-0 row).
++    GGML_ASSERT(row_bytes_src0 == row_bytes_src1);
++    GGML_ASSERT(row_bytes_src0 == row_bytes_dst);
++
++    int64_t o[4] = {0, 0, 0, 0};
++    o[dim] = src0->ne[dim];
++
++    // TODO: smarter multi-threading
++    for (int i3 = 0; i3 < ne3; i3++) {
++        for (int i2 = ith; i2 < ne2; i2 += nth) {
++            for (int i1 = 0; i1 < ne1; i1++) {
++                const char * x;
++                if (i1 < ne01 && i2 < ne02 && i3 < ne03) {
++                    x = (const char *)src0->data
++                      + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03;
++                } else {
++                    x = (const char *)src1->data
++                      + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;
++                }
++                char * y = (char *)dst->data
++                         + i1*nb1 + i2*nb2 + i3*nb3;
++                memcpy(y, x, row_bytes_dst);
++            }
++        }
++    }
++}
++
+ void ggml_compute_forward_concat(
+     const ggml_compute_params * params,
+     ggml_tensor * dst) {
+@@ -2102,7 +2215,16 @@ void ggml_compute_forward_concat(
+             } break;
+         default:
+             {
+-                ggml_compute_forward_concat_any(params, dst);
++                // opencoti F5 bug-226 fix (0035): block-quantized types
++                // (q8_0, q4_0, q5_0, …) route to concat_q which iterates
++                // over BLOCKS, not logical elements. Without this they
++                // hit concat_any whose per-element memcpy with type_size
++                // = block-size scrambles q8_0 K/V at M2 head-axis concat.
++                if (ggml_blck_size(src0->type) > 1) {
++                    ggml_compute_forward_concat_q(params, dst);
++                } else {
++                    ggml_compute_forward_concat_any(params, dst);
++                }
+             }
+     }
+ }

patches/0036-pcie-probe-consume.patch

@@ -0,0 +1,364 @@
+opencoti F5-opt W1 (#293) — PCIe/ReBAR profile boot-time reader + autodetect flags
+
+WHAT
+  Consumes the shipped rebar/PCIe probe (perf/llamafile/rebar-probe.{cu,sh}) at
+  server boot: reads the newest probe JSON (or $OPENCOTI_PCIE_PROFILE) for the
+  measured pinned H2D bandwidth + PCIe link width/speed, falling back to a
+  one-shot nvidia-smi query, then a conservative PCIe3 x8 default. The result
+  (effective GB/s + link geometry) is cached in a process-global pcie_profile,
+  logged at startup, and consumed later by M7 Rolling KV (#296) for streaming
+  tile sizing (tile_bytes_max = compute_ms * eff_bw_GBps * 0.8).
+
+  Adds:
+    - common/pcie-profile.{h,cpp}            (NEW — detection + global)
+    - common_params.pcie_autodetect/pcie_bw_gbps   (common.h)
+    - --pcie-autodetect / --pcie-bw-gbps     (common/arg.cpp)
+    - common/pcie-profile.cpp TU             (BUILD.mk COMMON_SRCS_CPP)
+    - pcie_profile_init() + boot log         (tools/server/server.cpp — hook)
+
+PROVENANCE (per opencoti upstream-sync directive)
+  opencoti-ORIGINAL — no code copied from upstream llama.cpp. The PCIe-link
+  sysfs read + ReBAR-active detection technique derives from lightseek's
+  tools/windows/rebar_test.cu (Windows NVML/VirtualAlloc original), Linux-ported
+  by opencoti in perf/llamafile/rebar-probe.cu (#293). The server-side reader
+  here parses that probe's JSON.
+  Upstream base: ggml-org/llama.cpp via Mozilla-Ocho/llamafile submodule
+  5e9c63546 (llamafile 0.10.1 pin). Apply order: after 0035 (independent —
+  common/ + server additive). Surgical hook: tools/server/server.cpp boot call
+  — registered in docs/protocols/UPSTREAM_SYNC.md.
+
+  Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
+
+diff -ruN state-pre-0036/llama.cpp/BUILD.mk state-post-0036/llama.cpp/BUILD.mk
+diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
+--- a/llama.cpp/BUILD.mk
++++ b/llama.cpp/BUILD.mk
+@@ -242,6 +242,7 @@ COMMON_SRCS_CPP := \
+ 	llama.cpp/common/ngram-cache.cpp \
+ 	llama.cpp/common/ngram-map.cpp \
+ 	llama.cpp/common/ngram-mod.cpp \
++	llama.cpp/common/pcie-profile.cpp \
+ 	llama.cpp/common/peg-parser.cpp \
+ 	llama.cpp/common/preset.cpp \
+ 	llama.cpp/common/reasoning-budget.cpp \
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1440,6 +1440,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             else throw std::invalid_argument("--neo-pipeline must be off|on|auto");
+         }
+     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE"));
++    // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md
++    add_opt(common_arg(
++        {"--pcie-autodetect"}, "MODE",
++        "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "on"  || value == "true"  || value == "1") params.pcie_autodetect = true;
++            else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false;
++            else throw std::invalid_argument("--pcie-autodetect must be on|off");
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT"));
++    add_opt(common_arg(
++        {"--pcie-bw-gbps"}, "F",
++        string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps),
++        [](common_params & params, const std::string & value) {
++            params.pcie_bw_gbps = std::stof(value);
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS"));
+     add_opt(common_arg(
+         {"--chunks"}, "N",
+         string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -445,6 +445,12 @@ struct common_params {
+     // active for the layer), 2 = auto (reserved for future load-based
+     // engagement; same as `on` today).
+     uint32_t neo_pipeline_mode    =     0;
++    // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md
++    // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7
++    // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON /
++    // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default).
++    bool    pcie_autodetect       =  true;
++    float   pcie_bw_gbps          =  0.0f;
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
+     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
+diff --git a/llama.cpp/common/pcie-profile.cpp b/llama.cpp/common/pcie-profile.cpp
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/common/pcie-profile.cpp
+@@ -0,0 +1,201 @@
++// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile.
++// See pcie-profile.h for the contract. CPU-side only; never throws.
++
++#include "pcie-profile.h"
++
++#include <algorithm>
++#include <cmath>
++#include <cstdio>
++#include <cstdlib>
++#include <cstring>
++#include <dirent.h>
++#include <fstream>
++#include <sstream>
++#include <string>
++#include <vector>
++
++namespace {
++
++// PCIe generation -> per-lane raw transfer rate (GT/s).
++float gen_to_gtps(int gen) {
++    switch (gen) {
++        case 1: return 2.5f;
++        case 2: return 5.0f;
++        case 3: return 8.0f;
++        case 4: return 16.0f;
++        case 5: return 32.0f;
++        case 6: return 64.0f;
++        default: return 0.0f;
++    }
++}
++
++// Effective usable H2D bandwidth from link geometry. The 0.8 factor folds in
++// 128b/130b (gen3+) encoding overhead + DMA/protocol inefficiency; matches the
++// rolling_kv.md auto-adapt formula. width lanes * GT/s / 8 bits-per-byte * 0.8.
++float geometry_bw_gbps(int width, float gtps) {
++    if (width <= 0 || gtps <= 0.0f) return 0.0f;
++    return (float) width * gtps * 0.8f / 8.0f;
++}
++
++// "8.0 GT/s PCIe" / "16.0 GT/s PCIe" -> leading float. Empty/garbage -> 0.
++float parse_leading_float(const std::string & s) {
++    return s.empty() ? 0.0f : (float) atof(s.c_str());
++}
++
++// Minimal value-after-key extraction for our OWN well-formed probe JSON.
++// Returns the substring between key's ':' and the next ',' or '}' / newline.
++bool json_raw_after(const std::string & body, const std::string & key, std::string & out) {
++    const size_t k = body.find(key);
++    if (k == std::string::npos) return false;
++    size_t c = body.find(':', k + key.size());
++    if (c == std::string::npos) return false;
++    ++c;
++    while (c < body.size() && (body[c] == ' ' || body[c] == '\t')) ++c;
++    size_t e = c;
++    while (e < body.size() && body[e] != ',' && body[e] != '\n' && body[e] != '}') ++e;
++    out = body.substr(c, e - c);
++    return true;
++}
++
++float json_float_after(const std::string & body, const std::string & key) {
++    std::string raw;
++    if (!json_raw_after(body, key, raw)) return 0.0f;
++    return (float) atof(raw.c_str());
++}
++
++// Quoted-string value: trims surrounding quotes/spaces.
++std::string json_string_after(const std::string & body, const std::string & key) {
++    std::string raw;
++    if (!json_raw_after(body, key, raw)) return {};
++    size_t a = raw.find('"');
++    if (a == std::string::npos) return {};
++    size_t b = raw.find('"', a + 1);
++    if (b == std::string::npos) return {};
++    return raw.substr(a + 1, b - a - 1);
++}
++
++std::string read_file(const std::string & path) {
++    std::ifstream f(path, std::ios::binary);
++    if (!f.is_open()) return {};
++    std::ostringstream ss;
++    ss << f.rdbuf();
++    return ss.str();
++}
++
++// Newest ./.opencoti/rebar-probe-*.json. Filenames embed a YYYYMMDD-HHMMSS
++// stamp, so lexicographic max == newest. Empty if none.
++std::string newest_probe_json() {
++    const char * env = getenv("OPENCOTI_PCIE_PROFILE");
++    if (env && env[0]) return std::string(env);
++
++    const char * dirs[] = {"./.opencoti", ".opencoti"};
++    std::string best;
++    for (const char * d : dirs) {
++        DIR * dp = opendir(d);
++        if (!dp) continue;
++        struct dirent * ent;
++        while ((ent = readdir(dp)) != nullptr) {
++            const std::string name = ent->d_name;
++            if (name.rfind("rebar-probe-", 0) != 0) continue;
++            if (name.size() < 5 || name.compare(name.size() - 5, 5, ".json") != 0) continue;
++            const std::string full = std::string(d) + "/" + name;
++            if (best.empty() || name > best.substr(best.find_last_of('/') + 1)) best = full;
++        }
++        closedir(dp);
++        if (!best.empty()) break;
++    }
++    return best;
++}
++
++// Parse a probe JSON into a profile. Prefers the MEASURED pinned H2D bandwidth
++// over the geometry formula; still records link width/speed for logging.
++bool profile_from_probe_json(const std::string & path, pcie_profile & out) {
++    const std::string body = read_file(path);
++    if (body.empty()) return false;
++
++    // measured pinned H2D: locate the "pinned" object, then its "h2d".
++    float pinned_h2d = 0.0f;
++    const size_t p = body.find("\"pinned\"");
++    if (p != std::string::npos) {
++        pinned_h2d = json_float_after(body.substr(p), "\"h2d\"");
++    }
++
++    const int   width = atoi(json_string_after(body, "\"pci_link_width_current\"").c_str());
++    const float gtps  = parse_leading_float(json_string_after(body, "\"pci_link_speed_current\""));
++
++    if (pinned_h2d <= 0.0f && width <= 0) return false;
++
++    out.link_width = width;
++    out.link_gtps  = gtps;
++    out.effective_bw_gbps = pinned_h2d > 0.0f ? pinned_h2d : geometry_bw_gbps(width, gtps);
++    out.source = "probe-json";
++    out.detail = "from " + path + (pinned_h2d > 0.0f ? " (measured pinned H2D)" : " (link geometry)");
++    return out.effective_bw_gbps > 0.0f;
++}
++
++// One-shot nvidia-smi query for link gen + width. CPU-side, no CUDA link dep.
++bool profile_from_nvidia_smi(pcie_profile & out) {
++    FILE * pp = popen("nvidia-smi --query-gpu=pcie.link.gen.current,pcie.link.width.current "
++                      "--format=csv,noheader,nounits -i 0 2>/dev/null", "r");
++    if (!pp) return false;
++    char buf[256] = {};
++    const bool got = fgets(buf, sizeof(buf), pp) != nullptr;
++    pclose(pp);
++    if (!got) return false;
++
++    int gen = 0, width = 0;
++    // format: "<gen>, <width>"
++    if (sscanf(buf, " %d , %d", &gen, &width) < 2) return false;
++    const float gtps = gen_to_gtps(gen);
++    const float bw = geometry_bw_gbps(width, gtps);
++    if (bw <= 0.0f) return false;
++
++    out.link_width = width;
++    out.link_gtps  = gtps;
++    out.effective_bw_gbps = bw;
++    out.source = "nvidia-smi";
++    out.detail = "gen" + std::to_string(gen) + " x" + std::to_string(width) + " (link geometry)";
++    return true;
++}
++
++pcie_profile g_pcie_profile;
++bool         g_pcie_inited = false;
++
++} // namespace
++
++pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps) {
++    pcie_profile prof;  // conservative default below
++
++    if (manual_gbps > 0.0f) {
++        prof.effective_bw_gbps = manual_gbps;
++        prof.source = "manual";
++        prof.detail = "--pcie-bw-gbps override";
++        return prof;
++    }
++
++    if (autodetect) {
++        const std::string js = newest_probe_json();
++        if (!js.empty() && profile_from_probe_json(js, prof)) return prof;
++        if (profile_from_nvidia_smi(prof)) return prof;
++    }
++
++    // Conservative default: PCIe 3.0 x8 (common on Ryzen 5600G/5700G hosts where
++    // the iGPU absorbs 8 lanes — see rolling_kv.md). ~6.4 GB/s.
++    prof.link_width = 8;
++    prof.link_gtps  = 8.0f;
++    prof.effective_bw_gbps = geometry_bw_gbps(8, 8.0f);
++    prof.source = "default";
++    prof.detail = autodetect ? "no probe JSON / nvidia-smi; assuming PCIe3 x8"
++                             : "autodetect disabled; assuming PCIe3 x8";
++    return prof;
++}
++
++const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps) {
++    g_pcie_profile = pcie_profile_detect(autodetect, manual_gbps);
++    g_pcie_inited = true;
++    return g_pcie_profile;
++}
++
++const pcie_profile & pcie_profile_get() {
++    return g_pcie_profile;
++}
+diff --git a/llama.cpp/common/pcie-profile.h b/llama.cpp/common/pcie-profile.h
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/common/pcie-profile.h
+@@ -0,0 +1,38 @@
++#pragma once
++
++// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile.
++//
++// Boot-time detection of the GPU's effective host-to-device transfer bandwidth
++// and PCIe link geometry. This is a foundation diagnostic and, more
++// importantly, the input M7 Rolling KV (#296) uses to size streaming KV tiles:
++//   tile_bytes_max = compute_ms_per_layer * effective_bw_gbps * 0.8
++// (see docs/features/rolling_kv.md, "Auto-adapt loop", step 1).
++//
++// Detection is CPU-side only (no CUDA dependency in common/). Sources, in
++// preference order:
++//   1. manual override  (--pcie-bw-gbps F, F > 0)
++//   2. explicit probe JSON via $OPENCOTI_PCIE_PROFILE
++//   3. newest ./.opencoti/rebar-probe-*.json (measured pinned H2D + link)
++//   4. one-shot `nvidia-smi` query (pcie.link.gen.current + width.current)
++//   5. conservative default (PCIe 3.0 x8 ~= 6.4 GB/s)
++// Never throws; never fails server boot.
++
++#include <string>
++
++struct pcie_profile {
++    float       effective_bw_gbps = 0.0f;       // usable H2D bandwidth estimate (GB/s)
++    int         link_width        = 0;          // current PCIe lane count (e.g. 8, 16)
++    float       link_gtps         = 0.0f;       // current per-lane transfer rate (GT/s)
++    std::string source            = "default";  // manual|probe-json|nvidia-smi|default
++    std::string detail;                         // human-readable note for logging
++};
++
++// Run detection once. manual_gbps > 0 forces source="manual"; autodetect=false
++// skips probe/nvidia-smi discovery and returns the conservative default.
++pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps);
++
++// Process-global: pcie_profile_init() runs detection, caches, and returns the
++// result; pcie_profile_get() returns the cached value (defaults until init
++// runs). Consumed by M7 Rolling KV (#296) at server boot.
++const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps);
++const pcie_profile & pcie_profile_get();
+diff --git a/llama.cpp/tools/server/server.cpp b/llama.cpp/tools/server/server.cpp
+--- a/llama.cpp/tools/server/server.cpp
++++ b/llama.cpp/tools/server/server.cpp
+@@ -7,6 +7,7 @@
+ #include "arg.h"
+ #include "build-info.h"
+ #include "common.h"
++#include "pcie-profile.h" // opencoti F5-opt W1 (#293)
+ #include "fit.h"
+ #include "llama.h"
+ #include "log.h"
+@@ -135,6 +136,17 @@ int server_main(int argc, char ** argv,
+     // struct that contains llama context and inference
+     server_context ctx_server;
+ 
++    // opencoti F5-opt W1 (#293): detect host<->device bandwidth + PCIe link
++    // once at boot; logged here and cached (pcie_profile_get) for M7 Rolling KV
++    // (#296) streaming-tile sizing. Never fails boot — falls back to a
++    // conservative PCIe3 x8 default on any detection miss.
++    {
++        const pcie_profile & pp = pcie_profile_init(params.pcie_autodetect, params.pcie_bw_gbps);
++        LOG_INF("%s: pcie profile: %.1f GB/s effective (x%d @ %.1f GT/s, source=%s) — %s\n",
++                __func__, (double) pp.effective_bw_gbps, pp.link_width, (double) pp.link_gtps,
++                pp.source.c_str(), pp.detail.c_str());
++    }
++
+     server_http_context ctx_http;
+     if (!ctx_http.init(params)) {
+         SRV_ERR("%s", "failed to initialize HTTP server\n");

patches/0042-fused-moe.patch

@@ -0,0 +1,527 @@
+opencoti F5-opt W3 (#292) — op-level fused MoE up+gate+GLU (GGML_OP_MOE_FUSED_UP_GATE)
+
+Adds a new ggml op, GGML_OP_MOE_FUSED_UP_GATE, that collapses a MoE FFN's two
+separate per-expert projections (up = up_exps @ cur, gate = gate_exps @ cur)
+plus the GLU activation into ONE op at decode (n_tokens == 1). The CUDA backend
+dispatches it straight to the existing fused mmvq kernel (ggml_cuda_mul_mat_vec_q
+with a fusion arg carrying the gate weight + glu_op), eliminating the gate_up
+intermediate's HBM round-trip and the standalone GLU kernel launch per MoE layer
+per decode token.
+
+Off by default. Engaged with --fused-moe-up-gate on (server) / cparams
+.fused_moe_up_gate / adapter fusedMoeUpGate. The build_moe_ffn graph hook only
+emits the op when the model presents SEPARATE up/gate expert weights
+(gate_exps && !gate_up_exps), same type, SILU|GELU activation, at decode — i.e.
+Qwen2-MoE / Qwen3-MoE / OLMoE / Mixtral-style layouts. Models that fuse gate+up
+at the weight level (a single ffn_gate_up_exps — Gemma-4 A4B, Qwen3.5-MoE)
+never match the hook and run unchanged. Flag-OFF is byte-identical to the pre-W3
+binary (op never emitted); flag-ON on a matching model is byte-identical to the
+unfused two-matmul + GLU path. Validated on OLMoE-1B-7B-0924-Instruct-Q4_K_M:
+graph-dump engagement confirmed (34 MOE_FUSED_UP_GATE nodes on CUDA0), greedy
+decode byte-identical OFF vs ON, +2.4% decode tok/s (265.9 -> 272.2).
+
+  opencoti-hook: f5-opt-fmoe — see docs/features/advanced_kv.md
+
+UPSTREAM PROVENANCE (for future re-sync)
+  Idea source : ik_llama.cpp op-level MoE fusion
+                https://github.com/ikawrakow/ik_llama.cpp
+                PR #229 (fused MoE mul_mat_id path), PR #520 (OOAE expert eval).
+                Commit context HEAD 8960c5ba5ee9db30ba838304373aa4dbec9f7cbd.
+                License MIT, Author Iwan Kawrakow.
+  Code vendored: NONE. This op is original opencoti code. ik_llama's
+                ggml_cuda_moe_up_gate_unary dispatches a fused mmvq-id kernel;
+                our tree already carries the equivalent primitive
+                (ggml_cuda_mul_mat_vec_q with ggml_cuda_mm_fusion_args.gate +
+                glu_op — ggml-cuda/mmvq.cu:555-582). W3 takes only the
+                op-level-fusion IDEA and routes our existing kernel through a
+                new ggml op + a build_moe_ffn graph hook. No ik_llama source is
+                copied into this patch.
+  ABI note    : GGML_OP_COUNT 96 -> 97. The cosmocc binary AND the ggml-cuda.so
+                DSO must be rebuilt together and paired (asymmetric DSO cache
+                hygiene). The CPU backend deliberately aborts the op (CUDA-only;
+                CPU MoE fusion is out of scope — only the CPU embedder matters
+                for CPU speed, and that is the separate W2 iqk FA engine).
+  Op convention (locked from ggml-cuda/mmvq.cu):
+    out = up_proj(src0=up_exps @ src1=cur) (.) act(gate_proj=fusion.gate @ cur)
+        == ggml_geglu_split(gate, up); the kernel reads the GLU variant from
+    fusion.glu_op (NOT dst op_params). New op srcs: src0=up_exps, src1=gate_exps,
+    src2=cur, src3=ids; op_params[0]=glu_op. CUDA dispatch passes src0=up_exps,
+    src1=cur(=src2), ids=src3, fusion.gate=gate_exps(=src1), glu_op=op_params[0].
+
+Files touched (all surgical; no new vendored files):
+  ggml/include/ggml.h            enum GGML_OP_MOE_FUSED_UP_GATE + ggml_moe_up_gate decl
+  ggml/src/ggml.c                NAME/SYMBOL entries, COUNT 96->97 asserts, creation fn
+  ggml/src/ggml-cpu/ggml-cpu.c   forward-dispatch + n_tasks: CUDA-only ABORT stub
+  ggml/src/ggml-cuda/ggml-cuda.cu eval dispatch (reuse fused mmvq) + supports_op gate
+  src/llama-cparams.h            bool fused_moe_up_gate
+  include/llama.h                bool fused_moe_up_gate (llama_context_params)
+  src/llama-context.cpp          param copy + default-params initializer
+  common/common.h                bool fused_moe_up_gate = false
+  common/common.cpp              cparams copy
+  common/arg.cpp                 --fused-moe-up-gate on|off (LLAMA_ARG_FUSED_MOE_UP_GATE)
+  src/llama-graph.cpp            build_moe_ffn hook: emit op for separate-up/gate decode
+
+---
+opencoti note (2026-05-31, bug-250): hunks regenerated via snapshot-diff against the post-0040 chain so the patch strict git-applies (3 common/* hunks had whitespace context drift). Content unchanged.
+diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp
+--- a/llama.cpp/common/arg.cpp
++++ b/llama.cpp/common/arg.cpp
+@@ -1375,6 +1375,91 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
+             params.slot_initial_ctx = value;
+         }
+     ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL}));
++    // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--rest-kv-eviction"},
++        {"--no-rest-kv-eviction"},
++        string_format("when context shift fires, evict the lowest-retention (KeyDiff key-similarity) contiguous window instead of the positional middle (default: %s)", params.rest_kv_eviction ? "enabled" : "disabled"),
++        [](common_params & params, bool value) {
++            params.rest_kv_eviction = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_EVICTION"));
++    add_opt(common_arg(
++        {"--rest-kv-recent"}, "N",
++        string_format("rest-kv: protect the most-recent N tokens from eviction (default: %d)", params.rest_kv_recent),
++        [](common_params & params, int value) {
++            params.rest_kv_recent = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_RECENT"));
++    add_opt(common_arg(
++        {"--rest-kv-layer"}, "N",
++        string_format("rest-kv: model layer whose keys score retention, -1 = auto/middle (default: %d)", params.rest_kv_layer),
++        [](common_params & params, int value) {
++            params.rest_kv_layer = value;
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER"));
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--headinfer-gpu-heads-frac"}, "F",
++        string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac),
++        [](common_params & params, const std::string & value) {
++            params.headinfer_gpu_heads_frac = std::stof(value);
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC"));
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--neo-pipeline"}, "MODE",
++        "neo (asymmetric GPU/CPU attention pipelining): off (default), on (engage when headinfer split is active for the layer), auto (reserved, same as on). Requires --headinfer-gpu-heads-frac < 1.0 to do anything.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "off")  params.neo_pipeline_mode = 0;
++            else if (value == "on")   params.neo_pipeline_mode = 1;
++            else if (value == "auto") params.neo_pipeline_mode = 2;
++            else throw std::invalid_argument("--neo-pipeline must be off|on|auto");
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE"));
++    // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--iqk-flash-attn"}, "MODE",
++        "iqk CPU flash-attention (vendored ik_llama.cpp engine): off (default), on. "
++        "Speeds up the M2 CPU-half attention on the token-generation path; falls back "
++        "to the mainline kernel for unsupported (type, shape) cases. x86_64-only.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "off" || value == "false" || value == "0") params.iqk_flash_attn = false;
++            else if (value == "on"  || value == "true"  || value == "1") params.iqk_flash_attn = true;
++            else throw std::invalid_argument("--iqk-flash-attn must be on|off");
++            ggml_iqk_flash_attn_set_enabled(params.iqk_flash_attn);
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_IQK_FLASH_ATTN"));
++    // opencoti F5-opt W3 (#292) fused-MoE — see docs/features/advanced_kv.md
++    add_opt(common_arg(
++        {"--fused-moe-up-gate"}, "MODE",
++        "fused MoE up+gate+GLU (CUDA decode): off (default), on. Collapses the "
++        "separate-up/gate expert matmuls + GLU into one op at decode so the CUDA "
++        "backend runs the fused mmvq kernel without graph-pattern fusion. Only "
++        "engages for separate-up/gate MoE models (e.g. Gemma-4 A4B) at n_tokens==1.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "off" || value == "false" || value == "0") params.fused_moe_up_gate = false;
++            else if (value == "on"  || value == "true"  || value == "1") params.fused_moe_up_gate = true;
++            else throw std::invalid_argument("--fused-moe-up-gate must be on|off");
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_FUSED_MOE_UP_GATE"));
++    // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md
++    add_opt(common_arg(
++        {"--pcie-autodetect"}, "MODE",
++        "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.",
++        [](common_params & params, const std::string & value) {
++            if      (value == "on"  || value == "true"  || value == "1") params.pcie_autodetect = true;
++            else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false;
++            else throw std::invalid_argument("--pcie-autodetect must be on|off");
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT"));
++    add_opt(common_arg(
++        {"--pcie-bw-gbps"}, "F",
++        string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps),
++        [](common_params & params, const std::string & value) {
++            params.pcie_bw_gbps = std::stof(value);
++        }
++    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS"));
+     add_opt(common_arg(
+         // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
+         {"--slot-shrink-idle-ms"}, "N",
+diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp
+--- a/llama.cpp/common/common.cpp
++++ b/llama.cpp/common/common.cpp
+@@ -1636,6 +1636,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
+     cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
+     // opencoti F5 M3 neo — see docs/features/advanced_kv.md
+     cparams.neo_pipeline_mode = params.neo_pipeline_mode;
++    // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md
++    cparams.fused_moe_up_gate = params.fused_moe_up_gate;
+     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+     cparams.n_batch           = params.n_batch;
+     cparams.n_ubatch          = params.n_ubatch;
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -445,6 +445,41 @@ struct common_params {
+     // active for the layer), 2 = auto (reserved for future load-based
+     // engagement; same as `on` today).
+     uint32_t neo_pipeline_mode    =     0;
++    // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md
++    // false = off (default; unfused 3-node path). true = fuse decode MoE
++    // up+gate+GLU into one GGML_OP_MOE_FUSED_UP_GATE op for the CUDA backend.
++    bool     fused_moe_up_gate    = false;
++    // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md
++    // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7
++    // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON /
++    // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default).
++    bool    pcie_autodetect       =  true;
++    // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md
++    // Engage the vendored ik_llama.cpp CPU Flash-Attention engine on the M2
++    // CPU-half (token-generation path). Off by default; flag-off is byte-identical.
++    // x86_64-only at runtime (the engine compiles only on the AVX2 x86_64 slice).
++    bool    iqk_flash_attn        = false;
++    float   pcie_bw_gbps          =  0.0f;
++    // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md
++    // Initial KV-cell allocation per sequence. 0 = full n_ctx_seq allocation
++    // (Phase 1 behavior — only zero-fill is deferred). Positive values cap
++    // the initial allocation and enable grow-on-demand up to n_ctx_seq.
++    int32_t slot_initial_ctx      =     0;
++    // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md
++    // Idle threshold (ms) after which a grown soft cap is shrunk back to
++    // slot_initial_ctx and the over-cap pages are decommitted via
++    // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior).
++    int32_t slot_shrink_idle_ms   =     0;
++    // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md
++    // Fraction of KV heads kept GPU-resident; the rest are offloaded to host
++    // memory and streamed back per step. 1.0 = off (no split). Only active for
++    // GPU-offloaded layers with flash-attention (non-transposed V cache).
++    float   headinfer_gpu_heads_frac = 1.0f;
++    // opencoti F5 M3 neo — see docs/features/advanced_kv.md
++    // 0 = off (default; M2 concat path runs), 1 = on (engage when split
++    // active for the layer), 2 = auto (reserved for future load-based
++    // engagement; same as `on` today).
++    uint32_t neo_pipeline_mode    =     0;
+     // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md
+     // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7
+     // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON /
+diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h
+--- a/llama.cpp/ggml/include/ggml.h
++++ b/llama.cpp/ggml/include/ggml.h
+@@ -583,6 +583,8 @@ extern "C" {
+ 
+         GGML_OP_GLU,
+ 
++        GGML_OP_MOE_FUSED_UP_GATE, // opencoti-hook: f5-opt-fmoe — W3 #292
++
+         GGML_OP_COUNT,
+     };
+ 
+@@ -1437,6 +1439,22 @@ extern "C" {
+             struct ggml_tensor  * b,
+             struct ggml_tensor  * ids);
+ 
++    // opencoti-hook: f5-opt-fmoe — W3 #292: op-level fused MoE up+gate+GLU.
++    // build_moe_ffn otherwise emits {mul_mat_id(up), mul_mat_id(gate), glu}
++    // as 3 nodes that the CUDA backend can only fuse via the fragile
++    // ggml_can_fuse_subgraph graph-pattern path (which does not engage at
++    // decode for Gemma-4). This single op lets the backend run the already
++    // existing fused mmvq kernel directly. Output is GLU(gate@cur) (.) (up@cur),
++    // matching ggml_geglu_split(gate, up). up_exps/gate_exps: expert weight
++    // stacks (same type); cur: input activations; ids: expert selection.
++    GGML_API struct ggml_tensor * ggml_moe_up_gate(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * up_exps,
++            struct ggml_tensor  * gate_exps,
++            struct ggml_tensor  * cur,
++            struct ggml_tensor  * ids,
++            enum   ggml_glu_op    glu_op);
++
+     // A: m columns, n rows,
+     // B: p columns, n rows,
+     // result is m columns, p rows
+diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
+--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
+@@ -2066,6 +2066,13 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
+             {
+                 ggml_compute_forward_glu(params, tensor);
+             } break;
++        case GGML_OP_MOE_FUSED_UP_GATE:
++            {
++                // opencoti-hook: f5-opt-fmoe — W3 #292. CUDA-only op. The
++                // build_moe_ffn graph hook emits it only when the CUDA backend
++                // owns the FFN nodes, so the CPU scheduler never reaches here.
++                GGML_ABORT("MOE_FUSED_UP_GATE has no CPU implementation (CUDA-only op)");
++            } break;
+         case GGML_OP_GET_REL_POS:
+             {
+                 ggml_compute_forward_get_rel_pos(params, tensor);
+@@ -2333,6 +2340,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
+                     GGML_ABORT("fatal error");
+             }
+             break;
++        case GGML_OP_MOE_FUSED_UP_GATE:
++            {
++                // opencoti-hook: f5-opt-fmoe — W3 #292. CUDA-only; never
++                // scheduled on CPU. Single task keeps the planner well-formed.
++                n_tasks = 1;
++            } break;
+         case GGML_OP_SILU_BACK:
+         case GGML_OP_MUL:
+         case GGML_OP_DIV:
+diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+@@ -3005,6 +3005,25 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+         case GGML_OP_MUL_MAT_ID:
+             ggml_cuda_mul_mat_id(ctx, dst);
+             break;
++        case GGML_OP_MOE_FUSED_UP_GATE:
++            {
++                // opencoti-hook: f5-opt-fmoe — W3 #292. Drive the existing fused
++                // mmvq-id kernel directly: up = src0 @ src1, gate = fusion.gate @
++                // src1, out = act(gate) (.) up. Byte-identical to the graph-pattern
++                // fusion path (see the { MUL_MAT_ID, MUL_MAT_ID, GLU } branch in
++                // the eval fusion loop). Decode-only (ne[2]==1): the build_moe_ffn
++                // hook only emits this at n_tokens==1 and supports_op enforces it.
++                const ggml_tensor * up_exps   = dst->src[0];
++                const ggml_tensor * gate_exps = dst->src[1];
++                const ggml_tensor * cur       = dst->src[2];
++                const ggml_tensor * ids       = dst->src[3];
++                GGML_ASSERT(dst->ne[2] == 1 && "MOE_FUSED_UP_GATE is decode-only (n_tokens==1)");
++                ggml_cuda_mm_fusion_args_host fusion_data{};
++                fusion_data.gate   = gate_exps;
++                fusion_data.glu_op = (ggml_glu_op) dst->op_params[0];
++                ggml_cuda_mul_mat_vec_q(ctx, up_exps, cur, ids, dst, &fusion_data);
++            }
++            break;
+         case GGML_OP_OUT_PROD:
+             ggml_cuda_out_prod(ctx, dst);
+             break;
+@@ -5210,6 +5229,44 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de
+                     return false;
+             }
+             break;
++        case GGML_OP_MOE_FUSED_UP_GATE:
++            {
++                // opencoti-hook: f5-opt-fmoe — W3 #292. Supported only in the
++                // exact shape the fused mmvq-id decode kernel handles: quantized
++                // up/gate expert stacks of identical type, F32 activations, I32
++                // ids, single-token (decode) output, non-split weights, and a GLU
++                // op the kernel implements explicitly (GEGLU/SWIGLU/SWIGLU_OAI —
++                // others silently degrade to a plain product). build_moe_ffn only
++                // emits the op when all of these hold, so supports_op and the hook
++                // stay consistent (the scheduler never strands it on the CPU).
++                const ggml_tensor * up_exps   = op->src[0];
++                const ggml_tensor * gate_exps = op->src[1];
++                const ggml_tensor * cur       = op->src[2];
++                const ggml_tensor * ids       = op->src[3];
++                if (!up_exps || !gate_exps || !cur || !ids) {
++                    return false;
++                }
++                if (up_exps->buffer && ggml_backend_buft_is_cuda_split(up_exps->buffer->buft)) {
++                    return false;
++                }
++                if (!ggml_is_quantized(up_exps->type) || up_exps->type != gate_exps->type) {
++                    return false;
++                }
++                if (cur->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) {
++                    return false;
++                }
++                if (op->ne[2] != 1) {
++                    return false;
++                }
++                switch ((ggml_glu_op) op->op_params[0]) {
++                    case GGML_GLU_OP_GEGLU:
++                    case GGML_GLU_OP_SWIGLU:
++                    case GGML_GLU_OP_SWIGLU_OAI:
++                        return true;
++                    default:
++                        return false;
++                }
++            }
+         case GGML_OP_MUL_MAT:
+         case GGML_OP_MUL_MAT_ID:
+             {
+diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c
+--- a/llama.cpp/ggml/src/ggml.c
++++ b/llama.cpp/ggml/src/ggml.c
+@@ -1078,9 +1078,10 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
+     "OPT_STEP_SGD",
+ 
+     "GLU",
++    "MOE_FUSED_UP_GATE",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
++static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
+ 
+ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "none",
+@@ -1188,9 +1189,10 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "sgd(x)",
+ 
+     "glu(x)",
++    "moe_up_gate(up,gate,x)",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
++static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
+ 
+ static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
+ 
+@@ -3314,6 +3316,53 @@ struct ggml_tensor * ggml_mul_mat_id(
+     return result;
+ }
+ 
++// ggml_moe_up_gate
++// opencoti-hook: f5-opt-fmoe — W3 #292. Op-level fusion of the two parallel
++// expert matmuls (up = up_exps @ cur, gate = gate_exps @ cur) plus the GLU
++// activation that build_moe_ffn otherwise emits as 3 separate graph nodes.
++// Output == ggml_geglu_split(gate, up) == act(gate) (.) up, where the CUDA
++// backend runs the existing fused mmvq kernel (act on the gate projection,
++// multiply by the up projection — see ggml-cuda/mmvq.cu). Mirrors
++// ggml_mul_mat_id's shape contract: up_exps is the primary "as" weight, cur
++// the "b" activations shared by both projections, ids the expert selection.
++
++struct ggml_tensor * ggml_moe_up_gate(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * up_exps,
++        struct ggml_tensor  * gate_exps,
++        struct ggml_tensor  * cur,
++        struct ggml_tensor  * ids,
++        enum   ggml_glu_op    glu_op) {
++    GGML_ASSERT(!ggml_is_transposed(up_exps));
++    GGML_ASSERT(!ggml_is_transposed(gate_exps));
++    GGML_ASSERT(ids->type == GGML_TYPE_I32);
++
++    // up and gate are parallel expert stacks: identical shape + type so one
++    // shared-activation fused mmvq can drive both weight sets.
++    GGML_ASSERT(ggml_are_same_shape(up_exps, gate_exps));
++    GGML_ASSERT(up_exps->type == gate_exps->type);
++
++    GGML_ASSERT(up_exps->ne[3] == 1);                 // 3d (one matrix per expert)
++    GGML_ASSERT(cur->ne[3] == 1);                     // cur is 3d
++    GGML_ASSERT(ids->ne[2] == 1 && ids->ne[3] == 1);  // ids is 2d
++    GGML_ASSERT(ids->ne[1] == cur->ne[2]);            // one expert list per cur row
++    GGML_ASSERT(up_exps->ne[0] == cur->ne[0]);        // can_mul_mat
++    GGML_ASSERT(ids->ne[0] % cur->ne[1] == 0);        // can broadcast
++
++    const int64_t ne[4] = { up_exps->ne[1], ids->ne[0], cur->ne[2], 1 };
++    struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
++
++    ggml_set_op_params_i32(result, 0, (int32_t) glu_op);
++
++    result->op     = GGML_OP_MOE_FUSED_UP_GATE;
++    result->src[0] = up_exps;
++    result->src[1] = gate_exps;
++    result->src[2] = cur;
++    result->src[3] = ids;
++
++    return result;
++}
++
+ // ggml_out_prod
+ 
+ static inline bool ggml_can_out_prod(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h
+--- a/llama.cpp/include/llama.h
++++ b/llama.cpp/include/llama.h
+@@ -362,6 +362,9 @@ extern "C" {
+         // 0 = off (default), 1 = on, 2 = auto. Effective only when the
+         // M2 head split is active for a given layer.
+         uint32_t neo_pipeline_mode;
++        // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md
++        // true = fuse decode MoE up+gate+GLU into one op. false = off (default).
++        bool     fused_moe_up_gate;
+         uint32_t n_rs_seq;          // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
+         int32_t  n_threads;         // number of threads to use for generation
+         int32_t  n_threads_batch;   // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -66,6 +66,8 @@ llama_context::llama_context(
+     cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac;
+     // opencoti F5 M3 neo — see docs/features/advanced_kv.md
+     cparams.neo_pipeline_mode = params.neo_pipeline_mode;
++    // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md
++    cparams.fused_moe_up_gate = params.fused_moe_up_gate;
+ 
+     cparams.n_threads        = params.n_threads;
+     cparams.n_threads_batch  = params.n_threads_batch;
+@@ -3359,6 +3361,7 @@ llama_context_params llama_context_default_params() {
+         /*.slot_shrink_idle_ms         =*/ 0,
+         /*.headinfer_gpu_heads_frac    =*/ 1.0f,
+         /*.neo_pipeline_mode            =*/ 0,  // opencoti F5 M3 neo — off by default
++        /*.fused_moe_up_gate           =*/ false, // opencoti F5 W3 #292 — off by default
+         /*.n_rs_seq                    =*/ 0,
+         /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
+         /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -32,6 +32,10 @@ struct llama_cparams {
+     // 2 = auto (same engagement as `on` for now — reserved for future
+     //         load-based engagement heuristics).
+     uint32_t neo_pipeline_mode;
++    // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md.
++    // true = build_moe_ffn collapses the up/gate expert matmuls + GLU into one
++    // GGML_OP_MOE_FUSED_UP_GATE op at decode (n_tokens==1). false = unfused path.
++    bool     fused_moe_up_gate;
+     uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+     int32_t  n_threads;       // number of threads to use for generation
+     int32_t  n_threads_batch; // number of threads to use for batch processing
+diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp
+--- a/llama.cpp/src/llama-graph.cpp
++++ b/llama.cpp/src/llama-graph.cpp
+@@ -1534,7 +1534,28 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
+     ggml_tensor * up = nullptr;
+     ggml_tensor * experts = nullptr;
+ 
+-    if (gate_up_exps) {
++    // opencoti-hook: f5-opt-fmoe — W3 #292: collapse the separate-up/gate
++    // expert matmuls + the GLU activation into ONE GGML_OP_MOE_FUSED_UP_GATE
++    // op at decode, so the CUDA backend runs the existing fused mmvq kernel
++    // without relying on graph-pattern fusion (ggml_can_fuse_subgraph, which
++    // does not engage for Gemma-4 at decode). Guards mirror supports_op:
++    // decode only (n_tokens==1), separate up/gate of identical type, no
++    // bias/scale, and GELU→GEGLU or SILU→SWIGLU. Off path is byte-identical.
++    bool fused_up_gate = false;
++    if (cparams.fused_moe_up_gate && gate_exps && !gate_up_exps &&
++            !up_exps_b && !gate_exps_b && !up_exps_s && !gate_exps_s &&
++            up_exps->type == gate_exps->type && n_tokens == 1 &&
++            (type_op == LLM_FFN_GELU || type_op == LLM_FFN_SILU)) {
++        const enum ggml_glu_op glu_op = (type_op == LLM_FFN_GELU)
++            ? GGML_GLU_OP_GEGLU : GGML_GLU_OP_SWIGLU;
++        cur = ggml_moe_up_gate(ctx0, up_exps, gate_exps, cur, selected_experts, glu_op);
++        cb(cur, "ffn_moe_fused_up_gate", il);
++        fused_up_gate = true;
++    }
++
++    if (fused_up_gate) {
++        // cur already holds act(gate) (.) up from the fused op
++    } else if (gate_up_exps) {
+         // merged gate_up path: one mul_mat_id, then split into gate and up views
+         ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts); // [n_ff*2, n_expert_used, n_tokens]
+         cb(gate_up, "ffn_moe_gate_up", il);
+@@ -1601,6 +1622,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
+ 
+     const bool has_gate = gate_exps || gate_up_exps;
+ 
++    // opencoti-hook: f5-opt-fmoe — W3 #292: the fused op already applied the
++    // GLU activation; skip the separate activation pass.
++    if (!fused_up_gate)
+     switch (type_op) {
+         case LLM_FFN_SILU:
+             if (gate_exps) {

patches/0071-state-io-cosmocc.patch

@@ -0,0 +1,88 @@
+From: opencoti
+Subject: [PATCH 0071] cosmocc state-IO open fix (bug-269)
+
+opencoti F5 M7 (#349, bug-269). Orthogonal to Rolling KV (0070) — captured as
+its own slot. Fixes ALL llama_state_*_save/load_file in the cosmocc/llamafile
+build, which were broken before this: llama.cpp/src/llama-mmap.cpp's COSMOCC
+llama_file constructor routed every open() through llamafile_open_gguf(), which
+pread()s the GGUF magic at offset 0 (EBADF on a write-only "wb" fd) and
+zip-routes any non-GGUF read; and write_raw's fwrite(fp) ran with fp==NULL.
+KV-state files are neither GGUF nor zip and are opened for write, so model
+load was unaffected but state save/load never worked through the binary (the
+M2 bug-222 "state round-trip" was thus never actually exercised end-to-end).
+
+FIX (cosmocc/host-only; no CUDA, no kernel):
+  * vendors/sources/llamafile/llamafile/llamafile.{c,h} (Mozilla-Ocho llamafile,
+    Apache-2.0): add llamafile_open_plain() — a plain fopen-backed handle with
+    no GGUF magic probe and no zip routing, for non-model files. It delegates
+    to the existing static llamafile_open_file().
+  * llama.cpp/src/llama-mmap.cpp: the COSMOCC llama_file ctor falls back to
+    llamafile_open_plain() when llamafile_open_gguf() returns NULL (models are
+    unaffected — they still open via the gguf path) and sets fp =
+    llamafile_fp(lfile) so write_raw has a valid stream.
+
+Provenance: opencoti-original. Touches Mozilla-Ocho llamafile.{c,h}
+(Apache-2.0) and upstream llama.cpp llama-mmap.cpp. Applies after 0070
+(independent file set — order immaterial); reverse-applies clean.
+diff --git a/llama.cpp/src/llama-mmap.cpp b/llama.cpp/src/llama-mmap.cpp
+--- a/llama.cpp/src/llama-mmap.cpp
++++ b/llama.cpp/src/llama-mmap.cpp
+@@ -186,11 +186,25 @@ struct llama_file::impl {
+ #else
+     impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
+ #ifdef COSMOCC
+-        // [llamafile] Use llamafile_open_gguf for all file opening
++        // [llamafile] Use llamafile_open_gguf for model files (raw .gguf,
++        // foo.zip@weights, /zip/ paths, and .llamafile archives via zip routing).
+         lfile = llamafile_open_gguf(fname, mode);
++        // opencoti bug-269: llamafile_open_gguf is model-oriented — it pread()s the
++        // GGUF magic at offset 0 and routes non-GGUF files into a zip parser. KV
++        // state files (llama_state_*_save_file) are neither GGUF nor zip and are
++        // opened for write ("wb" → pread EBADF). Both made every state save/restore
++        // fail in the cosmocc build. When the model opener declines, fall back to a
++        // plain fopen-backed handle (model loads are unaffected — they succeed above).
++        if (lfile == NULL) {
++            lfile = llamafile_open_plain(fname, mode);
++        }
+         if (lfile == NULL) {
+             throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+         }
++        // write_raw() writes via std::fwrite(fp); the cosmocc open path left fp NULL
++        // (read-only design). Expose the underlying FILE* so state-save writes land.
++        // Harmless for reads (read_raw_unsafe uses llamafile_read(lfile)).
++        fp = llamafile_fp(lfile);
+         size = llamafile_size(lfile);
+         return;
+ #endif
+diff --git a/llamafile/llamafile.c b/llamafile/llamafile.c
+--- a/llamafile/llamafile.c
++++ b/llamafile/llamafile.c
+@@ -318,6 +318,15 @@ struct llamafile *llamafile_open_gguf(const char *fname, const char *mode) {
+     return llamafile_open_zip(fname, 0, mode);
+ }
+ 
++// opencoti bug-269: plain fopen-backed handle for non-model files (KV state
++// save/load). llamafile_open_gguf() is model-oriented — it pread()s the GGUF
++// magic at offset 0 (which fails EBADF on a write-only "wb" fd) and routes any
++// non-GGUF file into a zip parser. State files are neither GGUF nor zip and are
++// opened for write, so they need a path with no magic probe and no zip routing.
++struct llamafile *llamafile_open_plain(const char *fname, const char *mode) {
++    return llamafile_open_file(fname, mode);
++}
++
+ FILE *llamafile_fp(struct llamafile *file) {
+     return file->fp;
+ }
+diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
+--- a/llamafile/llamafile.h
++++ b/llamafile/llamafile.h
+@@ -46,6 +46,9 @@ extern int FLAG_verbose;        // Verbose output (chatbot_main.cpp, metal.c, cu
+ 
+ struct llamafile;
+ struct llamafile *llamafile_open_gguf(const char *, const char *);  // UNUSED externally
++// opencoti bug-269: plain fopen-backed handle (no GGUF magic probe, no zip
++// routing) for non-model files such as KV state save/load.
++struct llamafile *llamafile_open_plain(const char *, const char *);
+ void llamafile_close(struct llamafile *);                           // UNUSED externally
+ long llamafile_read(struct llamafile *, void *, size_t);            // UNUSED externally
+ long llamafile_write(struct llamafile *, const void *, size_t);     // UNUSED externally

patches/0072-poly-kv-pool.patch

@@ -0,0 +1,98 @@
+From: opencoti
+Subject: [PATCH 0072] F5 M6 PolyKV S1 — SharedKVPool shared read-only prefix
+
+Server-side SharedKVPool (F5 M6 S1, #370): a read-only KV prefix shared across
+sequences/slots so N agents over one base context pay O(1) shared-prefix memory
+instead of O(N). Per-request opt-in via the `shared_pool_slot` (>=0 selects the
+pool donor slot) and `shared_prefix_n_tokens` task params; default-off (slot -1)
+is byte-identical to upstream slot handling. Pure server/task plumbing — no ggml
+or kernel change.
+
+Bench: perf/llamafile/advanced-kv-stack.bench.ts (A4/S'); gate .opencoti/m6-s1-shared-pool-gate.sh
+Milestone: F5 M6 PolyKV S1 — see docs/features/poly_kv.md
+Applies after: 0071 (numbered 0072 so lexical apply order lands it after the M7
+  rolling-KV 0070/0071 it composes with; the original 0060 slot predates 0070).
+
+diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp
+--- a/llama.cpp/tools/server/server-context.cpp
++++ b/llama.cpp/tools/server/server-context.cpp
+@@ -2907,6 +2907,48 @@ private:
+                             }
+                         }
+ 
++                        // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md
++                        // F5 M6 PolyKV S1: bit-share a pool slot's read-only prefix into this seq
++                        // (zero-copy cells.seq_add in unified mode) and skip re-processing it.
++                        // Caller contract: this request's prompt MUST begin with exactly the pool
++                        // slot's first shared_prefix_n_tokens tokens, and the pool slot must remain
++                        // resident while sharers reference it. RESIDENCY PREREQUISITE: the server
++                        // must be booted with --no-clear-idle. The upstream idle-slot prompt-cache
++                        // offload (clear_idle, default-on with --kv-unified + --cache-ram; PR #16391)
++                        // serializes an idle slot's KV to a host blob and CLEARS its device cells the
++                        // moment a new task launches — which would evict the pool prefix before any
++                        // sharer's seq_cp runs (seq would be empty → silent miss). We guard against
++                        // that below: if the pool's resident range does not cover [0,P), we skip the
++                        // share and fall through to normal full re-processing (correct, just slower).
++                        // We mirror the copy_state_to() precedent: after seq_cp we also set this
++                        // slot's token bookkeeping to the first P tokens, else keep_first(n_past)
++                        // asserts on the (empty) slot token list (server-common.cpp:397).
++                        if (slot.task->params.shared_pool_slot >= 0 &&
++                            slot.task->params.shared_prefix_n_tokens > 0 &&
++                            slot.task->params.shared_pool_slot != slot.id) {
++                            const int pool = slot.task->params.shared_pool_slot;
++                            int32_t   P    = std::min<int32_t>(slot.task->params.shared_prefix_n_tokens,
++                                                               (int32_t) input_tokens.size());
++                            auto * mem = llama_get_memory(ctx_tgt);
++                            // residency guard: the pool seq must actually hold cells covering [0,P)
++                            const llama_pos pool_max = llama_memory_seq_pos_max(mem, pool);
++                            if (pool_max < (llama_pos) (P - 1)) {
++                                SLT_WRN(slot, "poly-kv-pool: pool slot %d not resident to %d (max pos %d) — "
++                                              "skipping share, full reprocess (boot with --no-clear-idle)\n",
++                                        pool, (int) (P - 1), (int) pool_max);
++                            } else {
++                                // clear any stale KV on this slot, then bit-share the pool's [0,P) prefix
++                                llama_memory_seq_rm(mem, slot.id, -1, -1);
++                                llama_memory_seq_cp(mem, pool, slot.id, 0, P);
++                                // mirror token bookkeeping so keep_first(n_past)/pos_next stay consistent
++                                server_tokens shared = input_tokens.clone();
++                                shared.keep_first(P);
++                                slot.prompt.tokens = std::move(shared);
++                                n_past = P;
++                                SLT_INF(slot, "poly-kv-pool: shared %d-token prefix from slot %d (n_past -> %d)\n", (int) P, pool, n_past);
++                            }
++                        }
++
+                         // [TAG_PROMPT_LOGITS]
+                         if (n_past == slot.task->n_tokens() && n_past > 0) {
+                             SLT_WRN(slot, "need to evaluate at least 1 token for each active slot (n_past = %d, task.n_tokens() = %d)\n", n_past, slot.task->n_tokens());
+diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp
+--- a/llama.cpp/tools/server/server-task.cpp
++++ b/llama.cpp/tools/server/server-task.cpp
+@@ -273,6 +273,8 @@ task_params server_task::params_from_json_cmpl(
+     params.n_cmpl           = json_value(data,       "n_cmpl",             json_value(data, "n", 1));
+     params.n_cache_reuse    = json_value(data,       "n_cache_reuse",      defaults.n_cache_reuse);
+     params.session_id       = json_value(data,       "session_id",         std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md
++    params.shared_pool_slot       = json_value(data, "shared_pool_slot",       -1); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md
++    params.shared_prefix_n_tokens = json_value(data, "shared_prefix_n_tokens",  0); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md
+     //params.t_max_prompt_ms  = json_value(data,       "t_max_prompt_ms",    defaults.t_max_prompt_ms); // TODO: implement
+     params.t_max_predict_ms = json_value(data,       "t_max_predict_ms",   defaults.t_max_predict_ms);
+     params.response_fields  = json_value(data,       "response_fields",    std::vector<std::string>());
+diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h
+--- a/llama.cpp/tools/server/server-task.h
++++ b/llama.cpp/tools/server/server-task.h
+@@ -72,6 +72,15 @@ struct task_params {
+     // number of prompt tokens before the latest user message
+     int32_t n_before_user = -1;
+ 
++    // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md
++    // F5 M6 PolyKV S1 (SharedKVPool). When shared_pool_slot >= 0, the server
++    // bit-shares this request's [0, shared_prefix_n_tokens) KV with the pool
++    // slot's already-resident read-only prefix (zero-copy cells.seq_add in
++    // unified mode), so N agents share ONE physical copy of the prefix and the
++    // prefix is neither re-stored nor re-processed. <0 (default) is fully inert.
++    int32_t shared_pool_slot       = -1;
++    int32_t shared_prefix_n_tokens =  0;
++
+     int64_t t_max_prompt_ms  = -1; // TODO: implement
+     int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit
+ 

patches/0075-d512-turbo-vec.patch

@@ -0,0 +1,121 @@
+From: opencoti
+Subject: [PATCH 0075] F5 M6 PolyKV S2 — D=512 fused FA-VEC turbo2/turbo3 (Gemma-4 GLOBAL)
+
+#411. The Level-B fused FA-VEC turbo read (0073) covered head_dim {128,256} only;
+the Gemma-4 A4B GLOBAL layers use head_dim 512 (key_length=512) and therefore fell
+back to the Level-A turbo->f16 materialize — a decode tax that scales with the
+global-cache n_kv. This patch adds the head_dim-512 fused FA-VEC instances for
+turbo2/turbo3 so the global cache stays IN-REGISTER:
+
+  - fattn-vec.cuh: extern DECL_FATTN_VEC_CASE(512, turbo2/3) (the kernel is already
+    D-generic: nthreads/nthreads_V derive from D arithmetically, KQ smem ~4-8KB,
+    VKQ ~16 regs at ncols=1 — well within budget; verified compile + run clean).
+  - fattn.cu: turbo kernel-selection relax (Q->ne[0]==512 routes to VEC) + the two
+    D=512 FATTN_VEC_CASE dispatch lines for turbo2/turbo3.
+  - template-instances/fattn-vec-instance-turbo{2,3}_0: emit the D=512 case; the
+    autogenerator (generate_cu_files.py) emits the same line so re-gen stays exact.
+  - llama-graph.cpp: build_attn_mha fuse gate extended (fused_turbo_512) so D=512
+    turbo2/turbo3 decode ubatches fuse; turbo4/turbo8 @ 512 keep Level-A.
+
+Correctness is Parseval-EXACT: the WHT is block-diagonal over 128-blocks, so D=512
+decomposes into 4 independent Parseval-exact 128-blocks (WHT(Q/s).K_stored ==
+Q.K_true). Acceptance = LOGIT-EQUIVALENCE fused-vs-materialize (#355 harness),
+real_frac=0.0 for BOTH turbo3 and turbo2 (bit-equivalent, not merely close).
+Perf (the point): turbo3 @ ctx 27.7k, fused 41.0 vs materialize 29.7 tok/s = +38%,
+scaling with n_kv. Gate: .opencoti/p411-d512-gate.sh (+ p411-perf.sh).
+
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
+@@ -618,3 +618,8 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16)
+ EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_0)
+ EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_0)
+ EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO4_0)
++
++// opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY — Gemma-4 GLOBAL
++// layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256.
++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -337,6 +337,12 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0)
+ 
++    // opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL
++    // layers). turbo4/turbo8 keep no D=512 instance (Level-A materialize). Matches the fattn.cu
++    // kernel-selection relax + the L3 graph gate (both turbo2/3 + D==512 only).
++    FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0)
++    FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
++
+     GGML_ABORT("fatal error");
+ }
+ 
+@@ -474,7 +480,13 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
+     // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when
+     // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift.
+     if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0) {
+-        return can_use_vector_kernel ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE;
++        // opencoti #411 (F5 M6-S2 Level-B): turbo2/3 also ship a D=512 fused VEC instance for Gemma-4
++        // GLOBAL layers (key_length=512) so the 256k global cache reads in-register instead of the
++        // Level-A turbo→f16 materialize. turbo4/turbo8 have no D=512 instance → keep the ≤256 cap
++        // (Level-A fallback at 512). The L3 graph gate only emits turbo2/3 K/V to FA at 512 to match.
++        const bool turbo512_vec = Q->ne[0] == 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 &&
++            (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0);
++        return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE;
+     }
+ 
+     // If Turing tensor cores are available, use them:
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
+@@ -6,3 +6,4 @@
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
+@@ -6,3 +6,4 @@
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+@@ -79,6 +79,10 @@ for type_k in TYPES_KV:
+ for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]:
+     with open(f"fattn-vec-instance-{get_short_name(type_t)}-{get_short_name(type_t)}.cu", "w") as f:
+         f.write(SOURCE_FATTN_VEC.format(type_k=type_t, type_v=type_t))
++        # opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL
++        # layers, key_length=512). turbo4 has no D=512 instance (Level-A materialize fallback).
++        if type_t in ("GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0"):
++            f.write(f"DECL_FATTN_VEC_CASE(512, {type_t}, {type_t}); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512\n")
+ 
+ for ncols in [8, 16, 32, 64]:
+     for ncols2 in [1, 2, 4, 8, 16, 32]:
+diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp
+--- a/llama.cpp/src/llama-graph.cpp
++++ b/llama.cpp/src/llama-graph.cpp
+@@ -2019,10 +2019,17 @@ ggml_tensor * llm_graph_context::build_attn_mha(
+     }();
+     const int64_t tbv_n_stream    = k->ne[3] > 0 ? k->ne[3] : 1;
+     const int64_t tbv_nq_per_strm = q->ne[2] / tbv_n_stream;
++    // opencoti F5 M6-S2 Level-B #411: head_dim 512 (Gemma-4 GLOBAL layers, key_length=512) fuses
++    // for turbo2/turbo3 ONLY — the D=512 fused FA-VEC turbo instances added in #411 keep the 256k
++    // global cache in-register instead of the Level-A turbo→f16 materialize (a ~0.55× decode tax that
++    // scales with n_kv). turbo4/turbo8 at 512 still take the materialize fallback (no D=512 instance).
++    // The WHT is block-diagonal over 128-blocks, so 512 = 4 Parseval-exact 128-blocks → exact.
++    const bool fused_turbo_512 =
++        k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0);
+     const bool fused_turbo =
+         cparams.flash_attn && kq_b == nullptr &&
+         is_turbo_vec(k->type) && k->type == v->type &&
+-        (k->ne[0] == 128 || k->ne[0] == 256) &&
++        (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) &&
+         tbv_nq_per_strm <= tbv_fuse_max_nq;
+ 
+     if (fused_turbo) {

patches/0080-mtp-d512-fa.patch

@@ -0,0 +1,155 @@
+From: opencoti
+Subject: [PATCH 0080] Gemma-4 assistant-MTP D512/gqa<=2 flash-attn — real f16 VEC kernel (#491+#493, bug-567)
+
+The Gemma-4 E4B assistant-MTP drafter's single GLOBAL (non-SWA) layer is head_count 4 /
+head_count_kv 2 -> gqa_ratio 2 at head_dim 512. On the 0.10.3 base NEITHER the MMA
+(switch_ncols2, fattn.cu) NOR the TILE (switch_ncols2, fattn-tile.cuh) D512 path instantiates
+ncols2 in {1,2} (gated DV<=256 for binary size), so a D512/gqa<=2 flash-attn op aborts at
+runtime (bug-567). Every other Gemma-4 config takes ncols2 in {4,8} and is unaffected
+(A4B/12B/31B and the E4B base are gqa>=4; A4B globals are head16/kv2 -> gqa 8).
+
+#491 (host stop-gap): a guard in build_attn_mha (llama-graph.cpp) routed the D512/gqa<=2 layer
+to the EXISTING non-FA explicit-softmax branch. Correct but not a real flash-attn — "fa off" on
+the hot path.
+
+#493 (this patch, the real fix): add a REAL D512 f16 flash-attn via the VEC kernel, and narrow
+the #491 guard so f16/f32 K/V take it; the guard now fires ONLY for the residual with no D512
+kernel (quantized / bf16 K/V), which the assistant drafter never hits.
+  - template-instances/fattn-vec-instance-f16-f16.cu + generate_cu_files.py:
+      DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16) instance (the autogen line + the file).
+  - ggml-cuda/fattn-vec.cuh: extern decl for the D512 f16 instance.
+  - ggml-cuda/fattn.cu: (a) selector route in ggml_cuda_get_best_fattn_kernel — D512 && gqa_ratio<=2
+      && gqa_opt_applies && K/V f16 -> BEST_FATTN_KERNEL_VEC; (b) the matching VEC dispatch case
+      FATTN_VEC_CASE(512, F16, F16) in ggml_cuda_flash_attn_ext_vec, BEFORE the GGML_ABORT.
+      NOTE: fattn.cu has TWO independent head-size switches (selector + VEC dispatch); both need the
+      D512 case or the selector picks VEC but the dispatch falls through to a decode-time abort.
+  - llama.cpp/src/llama-graph.cpp: narrow the #491 guard — fa_no_gpu_kernel fires only when K/V are
+      NOT f16/f32 (fa_d512_f16_vec), so f16 D512/gqa<=2 now flash-attns on-GPU.
+
+The VEC kernel tiles Q->ne[1] (cols_per_block 1 decode / 2 prefill+verify) so the one f16 D512
+instance is a real flash-attn at ANY n_q (covers decode, prefill, and the spec-verify batch).
+logit_softcap is forced off for D!=128,256 in the VEC kernel; safe because Gemma-4 attn_soft_cap
+is false (softcap 0 -> use_logit_softcap false).
+
+Gate (.opencoti/m493-fa-d512-gate.sh): E4B -fa on assistant-MTP decodes 147.86 tps with NO abort;
+the verified-greedy output is BYTE-IDENTICAL to the #491 non-FA soft_max path and draft acceptance
+is unchanged (0.67910 both), so the new VEC FA is numerically correct (exact spec-decode validates
+every token, and a wrong FA would collapse acceptance). A4B (gqa 8, untouched MMA path) regresses
+nowhere. Both DSO paths rebuilt byte-identical (sha256 a8ddcd82).
+
+Marker convention: every changed file carries an `opencoti #491`/`opencoti #493` marker; see
+docs/protocols/UPSTREAM_SYNC.md. All hunks are nested (llama.cpp/<path>); no outer-vendor delta.
+
+diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp
+--- a/llama.cpp/src/llama-graph.cpp
++++ b/llama.cpp/src/llama-graph.cpp
+@@ -2122,7 +2122,29 @@
+ 
+     ggml_tensor * cur;
+ 
+-    const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
++    // opencoti F5 M6-S4 mtp (#441/#491): head_dim 512 has NO GPU flash-attn kernel for gqa_ratio ≤ 2.
++    // Both the MMA (ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2, fattn.cu) and TILE
++    // (launch_fattn_tile_switch_ncols2, fattn-tile.cuh) D512 paths only instantiate ncols2 ∈ {4,8}
++    // (gqa_ratio > 2); the ncols2 ∈ {1,2} variants are gated out for binary size (DV<=256 only), so a
++    // D512/gqa≤2 flash-attn op aborts at runtime (fattn.cu:132 / fattn-tile.cuh:1322). That is exactly
++    // the Gemma-4 E4B assistant-MTP drafter's single GLOBAL layer: head_count 4 / head_count_kv 2 →
++    // gqa_ratio 2. Route it through the EXISTING non-FA explicit-softmax branch below (any head_dim/gqa
++    // — see the "non-flash-attn branch" fallback note at the top of this fn). q/k here are post-permute
++    // (flash-attn layout), so q->ne[2]/k->ne[2] == the CUDA kernel's gqa_ratio and k->ne[0] == head_dim.
++    // NARROW: every working D512 model (A4B/E2B/12B/31B + the E4B base, gqa_ratio ≥ 4) keeps flash-attn,
++    // byte-unchanged; only the gqa≤2 case is special.
++    // opencoti #493 (F5 M6-S4): D512/gqa≤2 with f16 (or f32→f16-cast) K AND V now has a REAL GPU
++    // flash-attn — the f16 D512 VEC instance, which tiles Q->ne[1] (cols_per_block 1 decode / 2
++    // prefill+verify) so it computes at ANY n_q (see ggml_cuda_get_best_fattn_kernel, fattn.cu). So
++    // -fa on COMPUTES on-GPU for the E4B drafter (its KV is f16) instead of diverting to soft_max; no
++    // global fa-off, no non-FA fallback for the real path. The guard now fires ONLY for the residual
++    // with no D512/gqa≤2 kernel: QUANTIZED or bf16 K/V (the VEC D512 instance is f16-only; MMA/TILE
++    // abort, bug-567) — an exotic config the assistant drafter never hits. f16/f32 K/V → flash-attn.
++    const bool fa_d512_f16_vec = (k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32) &&
++                                 (v->type == GGML_TYPE_F16 || v->type == GGML_TYPE_F32);
++    const bool fa_no_gpu_kernel = k->ne[0] == 512 && k->ne[2] > 0 && (q->ne[2] / k->ne[2]) <= 2 &&
++                                  !fa_d512_f16_vec;
++    const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && !fa_no_gpu_kernel;
+     if (use_flash_attn) {
+         GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet");
+ 
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -360,6 +360,13 @@
+     FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0)
+     FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
+ 
++    // opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC dispatch — the runtime counterpart to the
++    // selector's D512/gqa<=2 f16 VEC route (ggml_cuda_get_best_fattn_kernel) + the f16-f16 D512
++    // instance. Without this case the selector picks VEC but the dispatch falls through to the abort
++    // below (bug-567 reappeared as a *decode-time* fatal at fattn.cu:363). The macro accepts F16 and
++    // F32→F16 for both K and V, matching the graph guard. Gemma-4 E4B assistant-MTP drafter global layer.
++    FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16)
++
+     GGML_ABORT("fatal error");
+ }
+ 
+@@ -513,6 +520,20 @@
+         return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE;
+     }
+ 
++    // opencoti #493 (F5 M6-S4): D512 (Gemma-4 GLOBAL layer) at gqa_ratio <= 2 — the E4B assistant-MTP
++    // drafter (head_count 4 / head_count_kv 2 → gqa 2) — has NO MMA/TILE kernel: switch_ncols2 aborts
++    // for DKQ>256 && gqa<=2 (the ncols2 ∈ {1,2} D512 instances are if-constexpr'd out, bug-567). The
++    // f16 D512 VEC instance (#493) tiles Q->ne[1] (cols_per_block 1 for decode, 2 for prefill/verify —
++    // ggml_cuda_flash_attn_ext_vec_case) so it is a REAL flash-attn at ANY n_q. Route f16/f16 D512 here
++    // when gqa<=2 so -fa on COMPUTES instead of diverting to the #491 non-FA host guard. D512/gqa>2
++    // (A4B / 12B / 31B / E4B base) keeps the faster MMA ncols2 ∈ {4,8} path — byte-unchanged. Past the
++    // case-512 check above, gqa_opt_applies already holds (so gqa_ratio>=2); gqa_ratio<=2 ⇒ exactly 2.
++    if (Q->ne[0] == 512 && gqa_ratio <= 2 && gqa_opt_applies &&
++        K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 &&
++        K->ne[1] % FATTN_KQ_STRIDE == 0) {
++        return BEST_FATTN_KERNEL_VEC;
++    }
++
+     // If Turing tensor cores are available, use them:
+     if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
+         if (can_use_vector_kernel) {
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
+@@ -631,3 +631,8 @@
+ // layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256.
+ extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
+ extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
++
++// opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC — Gemma-4 GLOBAL layer at gqa_ratio<=2 (the E4B
++// assistant-MTP drafter), where MMA/TILE abort (DKQ>256 && gqa<=2, bug-567). The VEC kernel tiles
++// Q->ne[1] so it is a real flash-attn at ANY n_q. Definition in fattn-vec-instance-f16-f16.cu.
++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu
+@@ -5,3 +5,4 @@
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
++DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16); // opencoti #493: Gemma-4 GLOBAL head_dim 512 f16, gqa<=2
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+@@ -74,6 +74,17 @@
+         with open(f"fattn-vec-instance-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f:
+             f.write(SOURCE_FATTN_VEC.format(type_k=type_k, type_v=type_v))
+ 
++# opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC for Gemma-4 GLOBAL layers at gqa_ratio<=2 (the E4B
++# assistant-MTP drafter's single global layer is head_count 4 / head_count_kv 2 → gqa 2). The MMA and
++# TILE D512 paths only instantiate ncols2 ∈ {4,8} (gqa>2); ncols2 ∈ {1,2} are if-constexpr'd out, so a
++# D512/gqa<=2 op aborts at runtime (fattn.cu switch_ncols2, bug-567). The VEC kernel tiles Q->ne[1]
++# (cols_per_block 1 for decode, 2 for prefill/verify — ggml_cuda_flash_attn_ext_vec_case) so it is a
++# REAL flash-attn at ANY n_q for D512. Mirrors #411's turbo512 but for plain f16 K/V → -fa on COMPUTES
++# instead of falling back to the #491 non-FA host guard. f16/f16 only (drafter KV is f16); quantized
++# D512/gqa<=2 keeps the guard (no VEC D512 quant instance).
++with open("fattn-vec-instance-f16-f16.cu", "a") as f:
++    f.write("DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16); // opencoti #493: Gemma-4 GLOBAL head_dim 512 f16, gqa<=2\n")
++
+ # opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec instances
+ # (K==V same tier only; the in-register readers live in fattn-turbo.cuh).
+ for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]:

patches/0082-cuda-build-parallelism.patch

@@ -0,0 +1,320 @@
+diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
+--- a/llamafile/cuda.sh
++++ b/llamafile/cuda.sh
+@@ -259,6 +259,22 @@
+     COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS"
+ fi
+ 
++# opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles
++# the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left
++# alone at the build tail uses one core while the rest idle. --threads compiles the arches
++# concurrently. To avoid oversubscribing the bulk (JOBS nvcc already saturate the cores), we divide
++# the outer job-slot count by NVCC_THREADS so PEAK concurrency stays ≈ JOBS (slots × threads). The
++# Windows cuda.bat already passes --threads 5; the Linux path lacked it. Tunable: NVCC_THREADS=1
++# disables (restores pre-#531 behavior). See docs/protocols/LLAMAFILE_UPGRADE.md.
++NVCC_THREADS="${NVCC_THREADS:-2}"
++JOBS_EFF="$JOBS"
++if [ "${NVCC_THREADS}" -gt 1 ] 2>/dev/null; then
++    COMMON_FLAGS="$COMMON_FLAGS --threads ${NVCC_THREADS}"
++    JOBS_EFF=$(( JOBS / NVCC_THREADS ))
++    [ "$JOBS_EFF" -lt 1 ] && JOBS_EFF=1
++fi
++echo "  nvcc --threads ${NVCC_THREADS}  →  ${JOBS_EFF} job-slots × ${NVCC_THREADS} arch-threads (#531)"
++
+ # Collect sources
+ collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_QUANTS"
+ echo "  Sources: $NUM_SOURCES .cu files"
+@@ -267,7 +283,7 @@
+ START_TIME=$(date +%s)
+ 
+ # Compile GPU sources
+-compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS"
++compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF"
+ 
+ COMPILE_TIME=$(date +%s)
+ echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds"
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+@@ -2600,3 +2600,41 @@
+ extern DECL_FATTN_MMA_F16_CASE(576, 512, 16,  4);
+ extern DECL_FATTN_MMA_F16_CASE(576, 512,  1, 32);
+ extern DECL_FATTN_MMA_F16_CASE(576, 512,  2, 32);
++
++// opencoti-hook: cuda-build-parallelism (#531) — shard the DCA-fused MMA case into parallel
++// instance TUs (template-instances/fattn-mma-f16-dca-fused-instance-*.cu), mirroring the stock
++// DECL_FATTN_MMA_F16_CASE sharding above. Before this, fattn.cu's dca-fused dispatch
++// (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch) instantiated all 48
++// ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case<DKQ,DV,ncols1,ncols2> INLINE — DKQ==DV ∈ {128,256,512}
++// × ncols2 ∈ {1,2,4,8} × ncols1 ∈ {8,16,32,64}/ncols2 → 576 device kernels in ONE TU = a ~52-min
++// SOLO compile tail (fattn.o was 29 MB, the largest object, finishing ~52 min after every other TU).
++// These extern decls move the instantiation to the parallel bulk; fattn.cu now just references the
++// symbols (the regular MMA/TILE/VEC cases were already sharded this way; only our DCA kernel wasn't).
++// The instance set matches the dispatch 1:1 — every reachable _case<DKQ,DV,ncols1,ncols2> has a
++// definition, none extra — so the linked DSO is byte-for-byte equivalent (same 576 kernels, relocated).
++#define DECL_FATTN_MMA_F16_DCA_FUSED_CASE(DKQ, DV, ncols1, ncols2)                 \
++    template void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case                  \
++    <DKQ, DV, ncols1, ncols2>(ggml_backend_cuda_context & ctx, ggml_tensor * dst)  \
++
++// DCA always uses DKQ==DV ∈ {128,256,512}; one extern per (ncols1,ncols2) across all three head dims.
++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(ncols1, ncols2)      \
++    extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, ncols1, ncols2);  \
++    extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, ncols1, ncols2);  \
++    extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, ncols1, ncols2);  \
++
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 1)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 1)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 1)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(64, 1)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 2)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 2)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 2)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 2)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 4)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 4)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 4)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 4)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 1, 8)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 8)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 8)
++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 8)
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 1);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 2);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 4);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 1, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 1, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 1, 8);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 4);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 8);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 1);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 2);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 2);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 4);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 8);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 64, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 64, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 64, 1);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 1);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 1);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 2);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 2);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 4);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 4);
+diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu
+new file mode 100644
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu
+@@ -0,0 +1,10 @@
++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of
++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail.
++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh.
++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128).
++
++#include "../fattn-mma-f16.cuh"
++
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 8);
++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 8);

patches/0083-tcq-warp-encode.patch

@@ -0,0 +1,868 @@
+diff --git a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu
+@@ -748,33 +748,17 @@
+     const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t);
+     const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0;
+ 
+-    // one-time per-device: prefer the 8 KB/block shared-memory backtrace (opt-in); else global fallback buffer
+-    static int  tcq3_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {};
+-    static bool tcq3_bt_checked   [GGML_CUDA_MAX_DEVICES] = {};
+-    constexpr int tcq3_bt_shared_bytes = 128 * 64;
+-    if (!tcq3_bt_checked[ctx.device]) {
+-        tcq3_bt_checked[ctx.device] = true;
+-        const char * env = getenv("TURBO_TCQ_SHARED_BT");
+-        if (!env || atoi(env) != 0) {
+-            int max_shared_optin = 0;
+-            CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device));
+-            if (max_shared_optin >= tcq3_bt_shared_bytes) {
+-                if (tcq_fp64_enabled())
+-                    CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo3_tcq<idx_t, double>), tcq3_bt_shared_bytes);
+-                else
+-                    CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo3_tcq<idx_t>, tcq3_bt_shared_bytes);
+-                tcq3_use_shared_bt[ctx.device] = 1;
+-            }
+-        }
+-    }
+-    if (!tcq3_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64);
++    // #533 warp-synchronous encode: bt is always a global per-group slab. With TCQ3_WPB groups/block a shared-bt
++    // would cost WPB*8KB; and the warp kernel is no longer barrier-bound, so global bt is free here. One 8KB slab
++    // per group (indexed by `group`, not blockIdx.x). tcq_bt_buf grows monotonically and is reused across launches.
++    ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64);
+ 
+     const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00);
+     const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01);
+     const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02);
+     const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
+     const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
+-    const int shared_bytes = tcq3_use_shared_bt[ctx.device] ? tcq3_bt_shared_bytes : 0;
++    const int tcq3_grid = (int)((ne_total_groups + TCQ3_WPB - 1) / TCQ3_WPB);  // #533 warp-packed: TCQ3_WPB groups/block, 32 thr/group
+     // opencoti-hook: TCQ activation-dump (F-D, #515) — bind the staging buffer for this launch (no-op unless
+     // TURBO_TCQ_DUMP_ACTS is set). tcq_dump_init() is idempotent; dump_dev is nullptr when inactive/cap-reached,
+     // in which case d_tcq_dump_buf is left at its nullptr default and the kernel's dump branch is a uniform no-op.
+@@ -782,15 +766,15 @@
+     float * dump_dev = tcq_dump_pre_launch((long) ne_total_groups);
+     if (dump_dev) CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_dump_buf, &dump_dev, sizeof(float*)));
+     if (tcq_fp64_enabled())
+-        k_set_rows_turbo3_tcq<idx_t, double><<<(int)ne_total_groups, 512, shared_bytes, stream>>>(
++        k_set_rows_turbo3_tcq<idx_t, double><<<tcq3_grid, TCQ3_WPB*32, 0, stream>>>(
+             src0_d, src1_d, (block_turbo3_tcq *)dst->data,
+-            ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13,
++            ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13,
+             s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3,
+             ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd);
+     else
+-        k_set_rows_turbo3_tcq<idx_t><<<(int)ne_total_groups, 512, shared_bytes, stream>>>(
++        k_set_rows_turbo3_tcq<idx_t><<<tcq3_grid, TCQ3_WPB*32, 0, stream>>>(
+             src0_d, src1_d, (block_turbo3_tcq *)dst->data,
+-            ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13,
++            ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13,
+             s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3,
+             ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd);
+     if (dump_dev) tcq_dump_post_launch((long) ne_total_groups, stream);
+@@ -827,42 +811,26 @@
+     const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t);
+     const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0;
+ 
+-    static int  tcq2_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {};
+-    static bool tcq2_bt_checked   [GGML_CUDA_MAX_DEVICES] = {};
+-    constexpr int tcq2_bt_shared_bytes = 128 * 64;
+-    if (!tcq2_bt_checked[ctx.device]) {
+-        tcq2_bt_checked[ctx.device] = true;
+-        const char * env = getenv("TURBO_TCQ_SHARED_BT");
+-        if (!env || atoi(env) != 0) {
+-            int max_shared_optin = 0;
+-            CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device));
+-            if (max_shared_optin >= tcq2_bt_shared_bytes) {
+-                if (tcq_fp64_enabled())
+-                    CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo2_tcq<idx_t, double>), tcq2_bt_shared_bytes);
+-                else
+-                    CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo2_tcq<idx_t>, tcq2_bt_shared_bytes);
+-                tcq2_use_shared_bt[ctx.device] = 1;
+-            }
+-        }
+-    }
+-    if (!tcq2_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64);
++    // #533 warp-synchronous encode: bt always a global per-group slab (TCQ2_WPB groups/block); warp kernel is
++    // no longer barrier-bound, so global bt is free. One 8KB slab per group, indexed by `group`.
++    ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64);
+ 
+     const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00);
+     const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01);
+     const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02);
+     const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
+     const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
+-    const int shared_bytes = tcq2_use_shared_bt[ctx.device] ? tcq2_bt_shared_bytes : 0;
++    const int tcq2_grid = (int)((ne_total_groups + TCQ2_WPB - 1) / TCQ2_WPB);  // #533 warp-packed: TCQ2_WPB groups/block, 32 thr/group
+     if (tcq_fp64_enabled())
+-        k_set_rows_turbo2_tcq<idx_t, double><<<(int)ne_total_groups, 256, shared_bytes, stream>>>(
++        k_set_rows_turbo2_tcq<idx_t, double><<<tcq2_grid, TCQ2_WPB*32, 0, stream>>>(
+             src0_d, src1_d, (block_turbo2_tcq *)dst->data,
+-            ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13,
++            ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13,
+             s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3,
+             ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd);
+     else
+-        k_set_rows_turbo2_tcq<idx_t><<<(int)ne_total_groups, 256, shared_bytes, stream>>>(
++        k_set_rows_turbo2_tcq<idx_t><<<tcq2_grid, TCQ2_WPB*32, 0, stream>>>(
+             src0_d, src1_d, (block_turbo2_tcq *)dst->data,
+-            ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13,
++            ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13,
+             s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3,
+             ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd);
+ }
+diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh
+@@ -109,11 +109,16 @@
+ static __device__ __forceinline__ float  tcq_acc_sqrt(float  v) { return sqrtf(v); }
+ static __device__ __forceinline__ double tcq_acc_sqrt(double v) { return sqrt(v);  }
+ 
+-// TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis
+-// 512 threads per block (one per trellis state), one block per 128-element group
+-// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384)
++// TCQ SET_ROWS encode: warp-synchronous Viterbi (#533). ncu proved the old 512-thread/__syncthreads design was
++// LATENCY-bound on the 128-step block-barrier chain (25% SM, 0.1% DRAM, 33% occ); #538's occupancy/shuffle tweak
++// couldn't help that. This collapses each 128-element group to ONE 32-thread warp (16 states/lane, thread-local
++// predecessor-min, __syncwarp broadcast, bt in global), packs TCQ3_WPB warps/block, and breaks the warp lockstep
++// so per-group serial latency is hidden by other warps. Validated byte-equivalent (round-trip decode cosine
++// 1.00000 vs the old design — .opencoti/tcq-warp-enc-full.cu) + 3.74x on the full encode. Reuses #538's warp-
++// shuffle min primitive. acc_t fp64 path (#506) preserved; bt is always global (use_shared_bt ignored).
++#define TCQ3_WPB 8
+ template<typename idx_t, typename acc_t = float>
+-static __global__ void __launch_bounds__(512, 1) k_set_rows_turbo3_tcq(
++static __global__ void __launch_bounds__(TCQ3_WPB*32, 3) k_set_rows_turbo3_tcq(
+         const float * __restrict__ src0, const idx_t * __restrict__ src1,
+         block_turbo3_tcq * __restrict__ dst, const int64_t ne_total_groups,
+         uint8_t * __restrict__ bt_buf,
+@@ -127,12 +132,13 @@
+         const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd,
+         const uint3 ne11_fd, const uint3 ne12_fd) {
+ 
+-    const int64_t group = blockIdx.x;
++    const int warp = threadIdx.x >> 5;
++    const int lane = threadIdx.x & 31;
++    const int64_t group = (int64_t)blockIdx.x * TCQ3_WPB + warp;
+     if (group >= ne_total_groups) return;
++    (void) use_shared_bt; (void) ne10; (void) ne13;   // bt is always global in the warp design
+ 
+-    const int sid = threadIdx.x; // state index 0..511
+-
+-    // Compute source and destination pointers (same index math as turbo3)
++    // Compute source and destination pointers (same index math as the 512-thread design; per-warp group)
+     const int64_t i_base = group * QK_TURBO3_TCQ;
+     uint32_t tmp = (uint32_t)i_base; uint2 div_mod;
+     div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x;
+@@ -145,244 +151,130 @@
+     block_turbo3_tcq * dst_blk = (block_turbo3_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3)
+                                   + (i00 / QK_TURBO3_TCQ);
+ 
+-    // Shared memory layout (~5KB, was ~35KB before global bt optimization):
+-    // x[128]     : rotated+normalized input (also reused as outputs[] after Viterbi)
+-    // cost[512]  : path costs buffer A (also reused for reductions)
+-    // cost_b[512]: path costs buffer B (double-buffering eliminates 2/3 of syncs)
+-    // Backtrace: one predecessor byte for each of the 64 low-state groups per
+-    // step. The predecessor is independent of the output bits in sid[8:6], so
+-    // storing 128×64 bytes is equivalent to the older 128×512 layout.
+-    extern __shared__ uint8_t bt_shared[];
+-    __shared__ acc_t x[128];
+-    __shared__ acc_t cost[512];
+-    __shared__ acc_t cost_b[512];
+-    __shared__ int warp_min_idx[16];
+-    __shared__ acc_t warp_min_cost[16];
+-    __shared__ acc_t pred_min_cost[64];
+-    __shared__ int shared_initial_state;
+-
+-    if (sid < 128) x[sid] = grp_src[sid];
+-    __syncthreads();
+-
+-    __syncthreads();
+-
+-    // Norm reduction
+-    cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f;
+-    __syncthreads();
+-    for (int stride = 256; stride >= 32; stride >>= 1) {
+-        if (sid < stride) cost[sid] += cost[sid + stride];
+-        __syncthreads();
+-    }
+-    if (sid < 32) {
+-        acc_t v = cost[sid];
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 16);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 8);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 4);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 2);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 1);
+-        if (sid == 0) cost[0] = v;
+-    }
+-    __syncthreads();
+-    acc_t grp_norm = tcq_acc_sqrt(cost[0]);
+-    acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f;
+-
+-    if (sid < 128) x[sid] *= inv_norm;
+-    __syncthreads();
+-
+-    // FWHT. The first five stages are contained within each 32-lane warp, so
+-    // use warp shuffles and only synchronize for the two cross-warp stages.
+-    if (sid < 128) {
+-        acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid];
+-        const int lane = sid & 31;
+-#pragma unroll
+-        for (int h = 1; h < 32; h <<= 1) {
+-            const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h);
+-            v = (lane & h) ? (other - v) : (v + other);
+-        }
+-        x[sid] = v;
+-    }
+-    __syncthreads();
+-    if (sid < 64) {
+-        const int j = ((sid >> 5) << 6) + (sid & 31);
+-        acc_t a = x[j], b = x[j + 32];
+-        x[j] = a + b; x[j + 32] = a - b;
+-    }
+-    __syncthreads();
+-    if (sid < 64) {
+-        acc_t a = x[sid], b = x[sid + 64];
+-        x[sid] = a + b; x[sid + 64] = a - b;
++    // Per-warp smem: whitened vec x[128] (reused as outputs[] after Viterbi) + pred_min_cost[64] broadcast +
++    // initial-state scratch. The Viterbi path-cost lives in registers (c[16], 16 states/lane). bt is global.
++    __shared__ acc_t x_s[TCQ3_WPB][128];
++    __shared__ acc_t pm_s[TCQ3_WPB][64];
++    __shared__ int   ini_s[TCQ3_WPB];
++    acc_t   * x = x_s[warp];
++    acc_t   * pred_min_cost = pm_s[warp];
++    uint8_t * bt = bt_buf + group * (int64_t)(128 * 64);   // global per-group backtrace slab
++
++    // --- load 4 elements/lane (stride-32 layout: lane owns positions L, L+32, L+64, L+96) ---
++    acc_t e0 = grp_src[lane], e1 = grp_src[lane+32], e2 = grp_src[lane+64], e3 = grp_src[lane+96];
++    // --- norm (warp-shuffle sum of squares) ---
++    acc_t ss = e0*e0 + e1*e1 + e2*e2 + e3*e3;
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xFFFFFFFFu, ss, o);
++    acc_t grp_norm = tcq_acc_sqrt(ss);
++    acc_t inv_norm = grp_norm > 1e-10f ? (acc_t)1.0 / grp_norm : (acc_t)0.0;
++    e0 *= inv_norm; e1 *= inv_norm; e2 *= inv_norm; e3 *= inv_norm;
++    // --- FWHT-128 (stride-32 layout: 5 shuffle-butterfly stages + 2 intra-thread cross stages,
++    //     byte-faithful to the 512-thread block FWHT) ---
++    e0 *= TURBO3_WHT_SIGNS1[lane];    e1 *= TURBO3_WHT_SIGNS1[lane+32];
++    e2 *= TURBO3_WHT_SIGNS1[lane+64]; e3 *= TURBO3_WHT_SIGNS1[lane+96];
++#pragma unroll
++    for (int h = 1; h < 32; h <<= 1) {
++        acc_t o0 = __shfl_xor_sync(0xFFFFFFFFu, e0, h); e0 = (lane & h) ? o0 - e0 : e0 + o0;
++        acc_t o1 = __shfl_xor_sync(0xFFFFFFFFu, e1, h); e1 = (lane & h) ? o1 - e1 : e1 + o1;
++        acc_t o2 = __shfl_xor_sync(0xFFFFFFFFu, e2, h); e2 = (lane & h) ? o2 - e2 : e2 + o2;
++        acc_t o3 = __shfl_xor_sync(0xFFFFFFFFu, e3, h); e3 = (lane & h) ? o3 - e3 : e3 + o3;
+     }
+-    __syncthreads();
++    { acc_t a = e0, b = e1; e0 = a + b; e1 = a - b; acc_t c = e2, d = e3; e2 = c + d; e3 = c - d; }  // cross-32: blk0<->1, blk2<->3
++    { acc_t a = e0, b = e2; e0 = a + b; e2 = a - b; acc_t c = e1, d = e3; e1 = c + d; e3 = c - d; }  // cross-64: blk0<->2, blk1<->3
+     constexpr float inv_sqrt_128 = 0.08838834764831845f;
+-    if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid];
+-    __syncthreads();
+-
+-    // opencoti-hook: TCQ activation-dump (F-D, #515). x[0..127] is now the post-FWHT unit-norm whitened vec —
+-    // exactly the xt the trellis quantizes against d_turbo3_tcq_codebook below, and x stays read-only through the
+-    // forward pass. Stage it for the F-E codebook retrain when enabled; nullptr (default) is a uniform no-op.
+-    // group == blockIdx.x < ne_total_groups, so [group*128 + sid] is in-bounds of the (ne_total_groups*128) buffer.
+-    if (d_tcq_dump_buf != nullptr && sid < 128) {
+-        d_tcq_dump_buf[group * 128 + sid] = (float) x[sid];
+-    }
+-
+-    if (sid == 0) cost[0] = grp_norm;
+-    __syncthreads();
+-
+-    acc_t saved_norm = cost[0];
+-
+-    // Viterbi forward pass: double-buffered cost (1 sync/step, was 3)
+-    uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64);
+-    cost[sid] = 0.0f;
+-    __syncthreads();
+-
++    e0 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane];    e1 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+32];
++    e2 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+64]; e3 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+96];
++    x[lane] = e0; x[lane+32] = e1; x[lane+64] = e2; x[lane+96] = e3;
++    __syncwarp();
++
++    // opencoti-hook: TCQ activation-dump (F-D, #515). e0..e3 are the post-FWHT unit-norm whitened vec.
++    if (d_tcq_dump_buf != nullptr) {
++        d_tcq_dump_buf[group*128 + lane]    = (float) e0;
++        d_tcq_dump_buf[group*128 + lane+32] = (float) e1;
++        d_tcq_dump_buf[group*128 + lane+64] = (float) e2;
++        d_tcq_dump_buf[group*128 + lane+96] = (float) e3;
++    }
++    acc_t saved_norm = grp_norm;
++
++    // --- Viterbi forward DP: warp-synchronous (16 states/lane, thread-local predecessor-min, __syncwarp). ---
++    // lane L owns states {16L..16L+15} = trellis-groups 2L (states 16L..16L+7) and 2L+1 (states 16L+8..16L+15);
++    // each group's 8 predecessors (g*8..g*8+7) fall entirely within lane g/2, so the predecessor-min is
++    // thread-local (no shuffle). A 64-entry per-warp smem broadcast under __syncwarp replaces the block barrier.
++    acc_t c[16];
++#pragma unroll
++    for (int i = 0; i < 16; i++) c[i] = 0.0f;
+     for (int t = 0; t < 128; t++) {
+-        // Double-buffer: even steps read cost/write cost_b, odd steps read cost_b/write cost
+-        acc_t * cost_rd = (t & 1) ? cost_b : cost;
+-        acc_t * cost_wr = (t & 1) ? cost   : cost_b;
+-
++        acc_t pm0 = c[0]; int pp0 = 0;
++#pragma unroll
++        for (int j = 1; j < 8; j++) { if (c[j]   < pm0) { pm0 = c[j];   pp0 = j; } }
++        acc_t pm1 = c[8]; int pp1 = 0;
++#pragma unroll
++        for (int j = 1; j < 8; j++) { if (c[8+j] < pm1) { pm1 = c[8+j]; pp1 = j; } }
++        pred_min_cost[2*lane]   = pm0; pred_min_cost[2*lane+1] = pm1;
++        bt[t*64 + 2*lane]       = (uint8_t) pp0; bt[t*64 + 2*lane+1] = (uint8_t) pp1;
++        __syncwarp();
+         acc_t xt = x[t];
+-
+-        // Right-shift trellis: ns = (prev >> 3) | (out << 6). The best
+-        // predecessor depends only on sid's low 6 bits, so compute those 64
+-        // minima once instead of repeating the same 8-way scan for each out.
+-        if (sid < 64) {
+-            const int base_prev = sid << 3;
+-            acc_t best = cost_rd[base_prev];
+-            int best_p = 0;
+-#pragma unroll
+-            for (int p = 1; p < 8; p++) {
+-                acc_t c = cost_rd[base_prev | p];
+-                if (c < best) {
+-                    best = c;
+-                    best_p = p;
+-                }
+-            }
+-            pred_min_cost[sid] = best;
+-            bt[t * 64 + sid] = (uint8_t) best_p;
+-        }
+-        __syncthreads();
+-
+-        const int pred_idx = sid & 0x3F;
+-        acc_t dist = xt - d_turbo3_tcq_codebook[sid];
+-        dist = dist * dist;
+-
+-        cost_wr[sid] = pred_min_cost[pred_idx] + dist;
+-        __syncthreads();
+-    }
+-    // After 128 steps (even count): final costs are in cost[] (step 127 writes to cost)
+-
+-    // Warp argmin over 512 costs
+-    {
+-        acc_t my_cost = cost[sid];
+-        int my_idx = sid;
+-        #pragma unroll
+-        for (int offset = 16; offset > 0; offset >>= 1) {
+-            acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset);
+-            int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset);
+-            if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; }
+-        }
+-        if (sid % 32 == 0) {
+-            warp_min_cost[sid / 32] = my_cost;
+-            warp_min_idx[sid / 32] = my_idx;
+-        }
++#pragma unroll
++        for (int i = 0; i < 16; i++) { int s = 16*lane + i; acc_t d = xt - d_turbo3_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; }
++        __syncwarp();
+     }
+-    __syncthreads();
+-    if (sid < 32) {
+-        acc_t best = (sid < 16) ? warp_min_cost[sid] : 3.4028234663852886e38f;
+-        int best_idx = (sid < 16) ? warp_min_idx[sid] : 0;
+-#pragma unroll
+-        for (int offset = 16; offset > 0; offset >>= 1) {
+-            acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset);
+-            int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset);
+-            if (other_cost < best) {
+-                best = other_cost;
+-                best_idx = other_idx;
+-            }
+-        }
+-        if (sid == 0) {
+-            shared_initial_state = best_idx; // temporarily: best final state (becomes initial after backtrack)
+-        }
++    // --- argmin over 512 states (per-lane min over 16, then warp-shuffle) ---
++    acc_t mc = c[0]; int mi = 16*lane;
++#pragma unroll
++    for (int i = 1; i < 16; i++) { if (c[i] < mc) { mc = c[i]; mi = 16*lane + i; } }
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) {
++        acc_t oc = __shfl_xor_sync(0xFFFFFFFFu, mc, o); int oi = __shfl_xor_sync(0xFFFFFFFFu, mi, o);
++        if (oc < mc || (oc == mc && oi < mi)) { mc = oc; mi = oi; }
+     }
+-    __syncthreads();
+-
+-    // Save x[] to global buffer before backtrack overwrites it
+-
+-    // Backtrack (inherently sequential, reads global bt)
++    // --- backtrack (lane 0; outputs reuse the x slab — x no longer needed) ---
+     uint8_t * outputs = (uint8_t *)x;
+-    if (sid == 0) {
+-        int state = shared_initial_state;
++    if (lane == 0) {
++        int state = mi;
+         for (int t = 127; t >= 0; t--) {
+             outputs[t] = (uint8_t)(state >> 6);
+-            int p = bt[t * 64 + (state & 0x3F)];
++            int p = bt[t*64 + (state & 0x3F)];
+             state = ((state & 0x3F) << 3) | p;
+         }
+-        shared_initial_state = state;
++        ini_s[warp] = state;
+     }
+-    __syncthreads();
++    __syncwarp();
++    int initial_state = ini_s[warp];
+ 
+-    // Save output symbols to global buffer
+-
+-    // Parallel recon norm: t>=2 can compute state directly from 3 outputs (3 shifts of 3 = 9 bits)
+-    acc_t my_recon_sq = 0.0f;
+-    if (sid < 128) {
+-        int cur_state;
+-        if (sid < 2) {
+-            cur_state = shared_initial_state;
+-            for (int t = 0; t <= sid; t++)
+-                cur_state = (cur_state >> 3) | (((int)outputs[t]) << 6);
+-        } else {
+-            cur_state = ((int)outputs[sid - 2] & 0x7)
+-                      | (((int)outputs[sid - 1] & 0x7) << 3)
+-                      | (((int)outputs[sid]     & 0x7) << 6);
+-        }
+-        acc_t c = d_turbo3_tcq_codebook[cur_state];
+-        my_recon_sq = c * c;
+-    }
+-    cost[sid] = my_recon_sq;
+-    __syncthreads();
+-    for (int stride = 256; stride >= 32; stride >>= 1) {
+-        if (sid < stride) cost[sid] += cost[sid + stride];
+-        __syncthreads();
+-    }
+-    if (sid < 32) {
+-        acc_t v = cost[sid];
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 16);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 8);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 4);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 2);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 1);
+-        if (sid == 0) cost[0] = v;
++    // --- recon norm (warp-shuffle sum; lane owns recon positions L, L+32, L+64, L+96) ---
++    acc_t recon_sq = 0.0f;
++#pragma unroll
++    for (int b = 0; b < 4; b++) {
++        int pos = lane + 32*b; int cur_state;
++        if (pos < 2) { cur_state = initial_state; for (int t = 0; t <= pos; t++) cur_state = (cur_state >> 3) | (((int)outputs[t]) << 6); }
++        else { cur_state = ((int)outputs[pos-2] & 0x7) | (((int)outputs[pos-1] & 0x7) << 3) | (((int)outputs[pos] & 0x7) << 6); }
++        acc_t cc = d_turbo3_tcq_codebook[cur_state]; recon_sq += cc * cc;
+     }
+-    __syncthreads();
+-    acc_t recon_norm = tcq_acc_sqrt(cost[0]);
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) recon_sq += __shfl_xor_sync(0xFFFFFFFFu, recon_sq, o);
++    acc_t recon_norm = tcq_acc_sqrt(recon_sq);
+     acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm;
+     corrected_norm *= innerq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v;
+ 
+-    // Parallel bitpack: qs stores 6 initial-state bits followed by 128 3-bit
+-    // output symbols. Each byte is independent, so avoid the old serial OR loop.
+-    if (sid < 49) {
+-        const int init_bits = (shared_initial_state >> 3) & 0x3F;
+-        uint8_t packed = 0;
+-#pragma unroll
+-        for (int bit = 0; bit < 8; bit++) {
+-            const int pos = sid * 8 + bit;
+-            int v = 0;
+-            if (pos < 6) {
+-                v = (init_bits >> pos) & 1;
+-            } else {
+-                const int sym_bit_pos = pos - 6;
+-                const int sym_idx = sym_bit_pos / 3;
+-                if (sym_idx < 128) {
+-                    v = (outputs[sym_idx] >> (sym_bit_pos % 3)) & 1;
+-                }
++    // --- bitpack: lane handles bytes {lane, lane+32} that are < 49 ---
++    const int init_bits = (initial_state >> 3) & 0x3F;
++#pragma unroll
++    for (int bb = 0; bb < 2; bb++) {
++        int byte = lane + 32*bb;
++        if (byte < 49) {
++            uint8_t packed = 0;
++#pragma unroll
++            for (int bit = 0; bit < 8; bit++) {
++                const int pos = byte*8 + bit; int v = 0;
++                if (pos < 6) v = (init_bits >> pos) & 1;
++                else { const int sym_bit_pos = pos - 6; const int sym_idx = sym_bit_pos / 3; if (sym_idx < 128) v = (outputs[sym_idx] >> (sym_bit_pos % 3)) & 1; }
++                packed |= (uint8_t)(v << bit);
+             }
+-            packed |= (uint8_t)(v << bit);
++            dst_blk->qs[byte] = packed;
+         }
+-        dst_blk->qs[sid] = packed;
+-    }
+-    if (sid == 0) {
+-        dst_blk->norm = __float2half((float)corrected_norm);
+     }
++    if (lane == 0) dst_blk->norm = __float2half((float)corrected_norm);
+ }
+ 
+ // TCQ GET_ROWS dequantize (for non-FA paths)
+@@ -455,10 +347,13 @@
+     -0.13432936f, -0.05269006f, +0.03536416f, +0.117640756f, -0.022776067f, +0.042032316f, +0.10472976f, +0.18042557f
+ };
+ 
+-// 2-bit TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis (k=2, L=8)
+-// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384)
++// 2-bit TCQ SET_ROWS encode: warp-synchronous Viterbi (#533, k=2, L=8, 256 states). Same warp redesign as
++// turbo3_tcq: 1 group / 32-thread warp (8 states/lane, thread-local quartet predecessor-min, __syncwarp
++// broadcast, bt in global), TCQ2_WPB warps/block. Replaces the old 256-thread/__syncthreads block (latency-
++// bound on the 128-step barrier chain). bt always global (use_shared_bt ignored); acc_t fp64 path preserved.
++#define TCQ2_WPB 8
+ template<typename idx_t, typename acc_t = float>
+-static __global__ void __launch_bounds__(256, 1) k_set_rows_turbo2_tcq(
++static __global__ void __launch_bounds__(TCQ2_WPB*32, 3) k_set_rows_turbo2_tcq(
+         const float * __restrict__ src0, const idx_t * __restrict__ src1,
+         block_turbo2_tcq * __restrict__ dst, const int64_t ne_total_groups,
+         uint8_t * __restrict__ bt_buf,
+@@ -472,12 +367,14 @@
+         const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd,
+         const uint3 ne11_fd, const uint3 ne12_fd) {
+ 
+-    const int grp = blockIdx.x;
+-    if (grp >= ne_total_groups) return;
+-    const int sid = threadIdx.x; // 0..255 = trellis state
++    const int warp = threadIdx.x >> 5;
++    const int lane = threadIdx.x & 31;
++    const int64_t group = (int64_t)blockIdx.x * TCQ2_WPB + warp;
++    if (group >= ne_total_groups) return;
++    (void) use_shared_bt; (void) ne10; (void) ne13;   // bt is always global in the warp design
+ 
+-    // Compute source and destination pointers (all threads, used by thread 0)
+-    const int64_t i_base = int64_t(grp) * QK_TURBO2_TCQ;
++    // Compute source and destination pointers (same index math as the 256-thread design; per-warp group)
++    const int64_t i_base = group * QK_TURBO2_TCQ;
+     uint32_t tmp = (uint32_t)i_base; uint2 div_mod;
+     div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x;
+     div_mod = fast_div_modulo(tmp, ne01_fd); const int64_t i01 = div_mod.y; tmp = div_mod.x;
+@@ -489,232 +386,120 @@
+     block_turbo2_tcq * dst_blk = (block_turbo2_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3)
+                                + (i00 / QK_TURBO2_TCQ);
+ 
+-    // Backtrace: one predecessor byte per 64 low-state groups per step.
+-    // The predecessor depends only on sid's low 6 bits (same as turbo3_tcq).
+-    extern __shared__ uint8_t bt_shared[];
+-    __shared__ acc_t x[128];
+-    __shared__ acc_t cost[256];
+-    __shared__ acc_t cost_b[256];   // double-buffering for Viterbi
+-    __shared__ int warp_min_idx[8];
+-    __shared__ acc_t warp_min_cost[8];
+-    __shared__ acc_t pred_min_cost[64];
+-    __shared__ int shared_initial_state;
+-
+-    if (sid < 128) x[sid] = grp_src[sid];
+-    __syncthreads();
+-
+-    __syncthreads();
+-
+-    // Norm reduction
+-    cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f;
+-    __syncthreads();
+-    for (int stride = 128; stride >= 32; stride >>= 1) {
+-        if (sid < stride) cost[sid] += cost[sid + stride];
+-        __syncthreads();
+-    }
+-    if (sid < 32) {
+-        acc_t v = cost[sid];
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 16);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 8);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 4);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 2);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 1);
+-        if (sid == 0) cost[0] = v;
+-    }
+-    __syncthreads();
+-    acc_t grp_norm = tcq_acc_sqrt(cost[0]);
+-    acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f;
+-
+-    if (sid < 128) x[sid] *= inv_norm;
+-    __syncthreads();
+-
+-    // FWHT. The first five stages use warp shuffles, the two cross-warp stages
+-    // use shared memory (same approach as turbo3_tcq).
+-    if (sid < 128) {
+-        acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid];
+-        const int lane = sid & 31;
+-#pragma unroll
+-        for (int h = 1; h < 32; h <<= 1) {
+-            const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h);
+-            v = (lane & h) ? (other - v) : (v + other);
+-        }
+-        x[sid] = v;
+-    }
+-    __syncthreads();
+-    if (sid < 64) {
+-        const int j = ((sid >> 5) << 6) + (sid & 31);
+-        acc_t a = x[j], b = x[j + 32];
+-        x[j] = a + b; x[j + 32] = a - b;
+-    }
+-    __syncthreads();
+-    if (sid < 64) {
+-        acc_t a = x[sid], b = x[sid + 64];
+-        x[sid] = a + b; x[sid + 64] = a - b;
++    // Per-warp smem: whitened vec x[128] (reused as outputs[] after Viterbi) + pred_min_cost[64] broadcast +
++    // initial-state scratch. Path-cost in registers (c[8], 8 states/lane). bt is global.
++    __shared__ acc_t x_s[TCQ2_WPB][128];
++    __shared__ acc_t pm_s[TCQ2_WPB][64];
++    __shared__ int   ini_s[TCQ2_WPB];
++    acc_t   * x = x_s[warp];
++    acc_t   * pred_min_cost = pm_s[warp];
++    uint8_t * bt = bt_buf + group * (int64_t)(128 * 64);   // global per-group backtrace slab
++
++    // --- load 4 elements/lane (stride-32 layout: lane owns positions L, L+32, L+64, L+96) ---
++    acc_t e0 = grp_src[lane], e1 = grp_src[lane+32], e2 = grp_src[lane+64], e3 = grp_src[lane+96];
++    // --- norm (warp-shuffle sum of squares) ---
++    acc_t ss = e0*e0 + e1*e1 + e2*e2 + e3*e3;
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xFFFFFFFFu, ss, o);
++    acc_t grp_norm = tcq_acc_sqrt(ss);
++    acc_t inv_norm = grp_norm > 1e-10f ? (acc_t)1.0 / grp_norm : (acc_t)0.0;
++    e0 *= inv_norm; e1 *= inv_norm; e2 *= inv_norm; e3 *= inv_norm;
++    // --- FWHT-128 (stride-32 layout: 5 shuffle-butterfly stages + 2 intra-thread cross stages) ---
++    e0 *= TURBO3_WHT_SIGNS1[lane];    e1 *= TURBO3_WHT_SIGNS1[lane+32];
++    e2 *= TURBO3_WHT_SIGNS1[lane+64]; e3 *= TURBO3_WHT_SIGNS1[lane+96];
++#pragma unroll
++    for (int h = 1; h < 32; h <<= 1) {
++        acc_t o0 = __shfl_xor_sync(0xFFFFFFFFu, e0, h); e0 = (lane & h) ? o0 - e0 : e0 + o0;
++        acc_t o1 = __shfl_xor_sync(0xFFFFFFFFu, e1, h); e1 = (lane & h) ? o1 - e1 : e1 + o1;
++        acc_t o2 = __shfl_xor_sync(0xFFFFFFFFu, e2, h); e2 = (lane & h) ? o2 - e2 : e2 + o2;
++        acc_t o3 = __shfl_xor_sync(0xFFFFFFFFu, e3, h); e3 = (lane & h) ? o3 - e3 : e3 + o3;
+     }
+-    __syncthreads();
++    { acc_t a = e0, b = e1; e0 = a + b; e1 = a - b; acc_t c = e2, d = e3; e2 = c + d; e3 = c - d; }  // cross-32
++    { acc_t a = e0, b = e2; e0 = a + b; e2 = a - b; acc_t c = e1, d = e3; e1 = c + d; e3 = c - d; }  // cross-64
+     constexpr float inv_sqrt_128 = 0.08838834764831845f;
+-    if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid];
+-    __syncthreads();
+-
+-    if (sid == 0) cost[0] = grp_norm;
+-    __syncthreads();
+-
+-    acc_t saved_norm = cost[0];
+-
+-    // Viterbi forward pass: double-buffered cost (1 sync/step, was 3)
+-    uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64);
+-    cost[sid] = 0.0f;
+-    __syncthreads();
+-
++    e0 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane];    e1 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+32];
++    e2 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+64]; e3 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+96];
++    x[lane] = e0; x[lane+32] = e1; x[lane+64] = e2; x[lane+96] = e3;
++    __syncwarp();
++    acc_t saved_norm = grp_norm;
++
++    // --- Viterbi forward DP: warp-synchronous (8 states/lane, thread-local quartet pred-min, __syncwarp). ---
++    // lane L owns states {8L..8L+7} = trellis-groups 2L (states 8L..8L+3) and 2L+1 (states 8L+4..8L+7); each
++    // group's 4 predecessors (g*4..g*4+3) fall within lane g/2, so predecessor-min is thread-local (no shuffle).
++    acc_t c[8];
++#pragma unroll
++    for (int i = 0; i < 8; i++) c[i] = 0.0f;
+     for (int t = 0; t < 128; t++) {
+-        acc_t * cost_rd = (t & 1) ? cost_b : cost;
+-        acc_t * cost_wr = (t & 1) ? cost   : cost_b;
+-
++        acc_t pm0 = c[0]; int pp0 = 0;
++#pragma unroll
++        for (int j = 1; j < 4; j++) { if (c[j]   < pm0) { pm0 = c[j];   pp0 = j; } }
++        acc_t pm1 = c[4]; int pp1 = 0;
++#pragma unroll
++        for (int j = 1; j < 4; j++) { if (c[4+j] < pm1) { pm1 = c[4+j]; pp1 = j; } }
++        pred_min_cost[2*lane]   = pm0; pred_min_cost[2*lane+1] = pm1;
++        bt[t*64 + 2*lane]       = (uint8_t) pp0; bt[t*64 + 2*lane+1] = (uint8_t) pp1;
++        __syncwarp();
+         acc_t xt = x[t];
+-
+-        // Right-shift trellis (k=2, L=8): ns = (prev >> 2) | (out << 6).
+-        // The best predecessor depends only on sid's low 6 bits, so compute
+-        // those 64 minima once instead of repeating the same 4-way scan per state.
+-        if (sid < 64) {
+-            const int base_prev = sid << 2;
+-            acc_t best = cost_rd[base_prev];
+-            int best_p = 0;
+-#pragma unroll
+-            for (int p = 1; p < 4; p++) {
+-                acc_t c = cost_rd[base_prev | p];
+-                if (c < best) {
+-                    best = c;
+-                    best_p = p;
+-                }
+-            }
+-            pred_min_cost[sid] = best;
+-            bt[t * 64 + sid] = (uint8_t) best_p;
+-        }
+-        __syncthreads();
+-
+-        const int pred_idx = sid & 0x3F;
+-        acc_t dist = xt - d_turbo2_tcq_codebook[sid];
+-        dist = dist * dist;
+-
+-        cost_wr[sid] = pred_min_cost[pred_idx] + dist;
+-        __syncthreads();
+-    }
+-    // After 128 steps (even count): final costs in cost[]
+-
+-    // Warp argmin over 256 costs
+-    {
+-        acc_t my_cost = cost[sid];
+-        int my_idx = sid;
+-        #pragma unroll
+-        for (int offset = 16; offset > 0; offset >>= 1) {
+-            acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset);
+-            int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset);
+-            if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; }
+-        }
+-        if (sid % 32 == 0) {
+-            warp_min_cost[sid / 32] = my_cost;
+-            warp_min_idx[sid / 32] = my_idx;
+-        }
++#pragma unroll
++        for (int i = 0; i < 8; i++) { int s = 8*lane + i; acc_t d = xt - d_turbo2_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; }
++        __syncwarp();
+     }
+-    __syncthreads();
+-    if (sid < 32) {
+-        acc_t best = (sid < 8) ? warp_min_cost[sid] : 3.4028234663852886e38f;
+-        int best_idx = (sid < 8) ? warp_min_idx[sid] : 0;
+-#pragma unroll
+-        for (int offset = 16; offset > 0; offset >>= 1) {
+-            acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset);
+-            int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset);
+-            if (other_cost < best) {
+-                best = other_cost;
+-                best_idx = other_idx;
+-            }
+-        }
+-        if (sid == 0) {
+-            shared_initial_state = best_idx;
+-        }
++    // --- argmin over 256 states (per-lane min over 8, then warp-shuffle) ---
++    acc_t mc = c[0]; int mi = 8*lane;
++#pragma unroll
++    for (int i = 1; i < 8; i++) { if (c[i] < mc) { mc = c[i]; mi = 8*lane + i; } }
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) {
++        acc_t oc = __shfl_xor_sync(0xFFFFFFFFu, mc, o); int oi = __shfl_xor_sync(0xFFFFFFFFu, mi, o);
++        if (oc < mc || (oc == mc && oi < mi)) { mc = oc; mi = oi; }
+     }
+-    __syncthreads();
+-
+-    // Save x[] to global buffer before backtrack overwrites it
+-
+-    // Backtrack (inherently sequential, reads compressed bt)
++    // --- backtrack (lane 0; outputs reuse the x slab) — turbo2: 2-bit output (state>>6), <<2 trellis shift ---
+     uint8_t * outputs = (uint8_t *)x;
+-    if (sid == 0) {
+-        int state = shared_initial_state;
++    if (lane == 0) {
++        int state = mi;
+         for (int t = 127; t >= 0; t--) {
+             outputs[t] = (uint8_t)(state >> 6);
+-            int p = bt[t * 64 + (state & 0x3F)];
++            int p = bt[t*64 + (state & 0x3F)];
+             state = ((state & 0x3F) << 2) | p;
+         }
+-        shared_initial_state = state;
++        ini_s[warp] = state;
+     }
+-    __syncthreads();
+-
+-    // Save output symbols to global buffer
++    __syncwarp();
++    int initial_state = ini_s[warp];
+ 
+-    // Parallel recon norm: t>=3 can compute state directly from 4 outputs (4 shifts of 2 = 8 bits)
+-    acc_t my_recon_sq = 0.0f;
+-    if (sid < 128) {
+-        int cur_state;
+-        if (sid < 3) {
+-            cur_state = shared_initial_state;
+-            for (int t = 0; t <= sid; t++)
+-                cur_state = (cur_state >> 2) | (((int)outputs[t]) << 6);
+-        } else {
+-            cur_state = ((int)outputs[sid - 3] & 0x3)
+-                      | (((int)outputs[sid - 2] & 0x3) << 2)
+-                      | (((int)outputs[sid - 1] & 0x3) << 4)
+-                      | (((int)outputs[sid]     & 0x3) << 6);
+-        }
+-        acc_t c = d_turbo2_tcq_codebook[cur_state];
+-        my_recon_sq = c * c;
+-    }
+-    cost[sid] = my_recon_sq;
+-    __syncthreads();
+-    for (int stride = 128; stride >= 32; stride >>= 1) {
+-        if (sid < stride) cost[sid] += cost[sid + stride];
+-        __syncthreads();
+-    }
+-    if (sid < 32) {
+-        acc_t v = cost[sid];
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 16);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 8);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 4);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 2);
+-        v += __shfl_down_sync(0xFFFFFFFFULL, v, 1);
+-        if (sid == 0) cost[0] = v;
++    // --- recon norm (warp-shuffle sum; turbo2: pos<3 walk, else 4 symbols x 2-bit) ---
++    acc_t recon_sq = 0.0f;
++#pragma unroll
++    for (int b = 0; b < 4; b++) {
++        int pos = lane + 32*b; int cur_state;
++        if (pos < 3) { cur_state = initial_state; for (int t = 0; t <= pos; t++) cur_state = (cur_state >> 2) | (((int)outputs[t]) << 6); }
++        else { cur_state = ((int)outputs[pos-3] & 0x3) | (((int)outputs[pos-2] & 0x3) << 2) | (((int)outputs[pos-1] & 0x3) << 4) | (((int)outputs[pos] & 0x3) << 6); }
++        acc_t cc = d_turbo2_tcq_codebook[cur_state]; recon_sq += cc * cc;
+     }
+-    __syncthreads();
+-    acc_t recon_norm = tcq_acc_sqrt(cost[0]);
++#pragma unroll
++    for (int o = 16; o > 0; o >>= 1) recon_sq += __shfl_xor_sync(0xFFFFFFFFu, recon_sq, o);
++    acc_t recon_norm = tcq_acc_sqrt(recon_sq);
+     acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm;
+     corrected_norm *= iq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v;
+ 
+-    // Parallel bitpack: qs stores 6 initial-state bits followed by 128 2-bit
+-    // output symbols. Each byte is independent (2-bit symbols never cross byte
+-    // boundaries after the 6-bit prefix), so avoid the serial OR loop.
+-    if (sid < 33) {
+-        const int init_bits = (shared_initial_state >> 2) & 0x3F;
+-        uint8_t packed = 0;
+-#pragma unroll
+-        for (int bit = 0; bit < 8; bit++) {
+-            const int pos = sid * 8 + bit;
+-            int v = 0;
+-            if (pos < 6) {
+-                v = (init_bits >> pos) & 1;
+-            } else {
+-                const int sym_bit_pos = pos - 6;
+-                const int sym_idx = sym_bit_pos / 2;
+-                if (sym_idx < 128) {
+-                    v = (outputs[sym_idx] >> (sym_bit_pos % 2)) & 1;
+-                }
++    // --- bitpack: 6 init bits + 128 x 2-bit symbols = 33 bytes; lane handles bytes {lane, lane+32} < 33 ---
++    const int init_bits = (initial_state >> 2) & 0x3F;
++#pragma unroll
++    for (int bb = 0; bb < 2; bb++) {
++        int byte = lane + 32*bb;
++        if (byte < 33) {
++            uint8_t packed = 0;
++#pragma unroll
++            for (int bit = 0; bit < 8; bit++) {
++                const int pos = byte*8 + bit; int v = 0;
++                if (pos < 6) v = (init_bits >> pos) & 1;
++                else { const int sym_bit_pos = pos - 6; const int sym_idx = sym_bit_pos / 2; if (sym_idx < 128) v = (outputs[sym_idx] >> (sym_bit_pos % 2)) & 1; }
++                packed |= (uint8_t)(v << bit);
+             }
+-            packed |= (uint8_t)(v << bit);
++            dst_blk->qs[byte] = packed;
+         }
+-        dst_blk->qs[sid] = packed;
+-    }
+-    if (sid == 0) {
+-        dst_blk->norm = __float2half((float)corrected_norm);
+     }
++    if (lane == 0) dst_blk->norm = __float2half((float)corrected_norm);
+ }
+ 
+ // 2-bit TCQ GET_ROWS dequantize

patches/0084-cuda-graphs.patch

@@ -0,0 +1,23 @@
+diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
+--- a/llamafile/cuda.sh
++++ b/llamafile/cuda.sh
+@@ -259,6 +259,19 @@
+     COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS"
+ fi
+ 
++# opencoti-hook: cuda-graphs (#533 decode-tps lever L6) — enable CUDA graph capture+replay.
++# Upstream llama.cpp's CMake defines GGML_CUDA_USE_GRAPHS by default; the llamafile cuda.sh build
++# omitted it, so the entire `#ifdef USE_CUDA_GRAPH` capture/replay path (ggml-cuda.cu:3318/4478/
++# 4515/4593/4698; common.cuh:1218) was compiled OUT — every decode step launched hundreds of
++# kernels individually with full per-launch CPU overhead on a latency-bound (~4.5% SM) decode.
++# This flag turns it on; the runtime guards (split buffers, unsupported node types, the opencoti
++# NEO head-split force-disable) fall back to eager capture-by-capture when a graph isn't viable, so
++# it composes with the opencoti stack. Tunable: OPENCOTI_NO_CUDA_GRAPHS=1 omits it (pre-#533).
++# See docs/evaluations/decode-levers.md (L6).
++if [ "${OPENCOTI_NO_CUDA_GRAPHS:-0}" != "1" ]; then
++    COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_USE_GRAPHS"
++fi
++
+ # opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles
+ # the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left
+ # alone at the build tail uses one core while the rest idle. --threads compiles the arches

patches/0085-dca-quant-prefill-inlaunch.patch

@@ -0,0 +1,237 @@
+From: opencoti
+Subject: [PATCH 0085] DCA quant-KV prefill — in-launch single-pass dequant (#554)
+
+Move scalar/turbo K/V dequant for the multi-chunk DCA fused flash-attn op out of the
+two-stage whole-cache graph lift (dca_lift_to_f16: q->f32->f16 materialized every
+forward, GPU-memory-bound and O(cache) — catastrophic at >=512k) and into a single-pass
+to_fp16_nc pool convert inside the fused launch (mirrors the non-DCA launch_fattn fast
+path). Byte-identical to the prior lift (both compute d*q in f32 then round to f16);
+restores DCA + quantized-KV as a first-class long-context path (~12x at 85k, ~30x at
+512k: 46 -> 1379+ tok/s, the 1M q5_0/q4_0 path). Also moves the regime==3 fused
+early-out above the f16-K asserts in ggml_cuda_flash_attn_ext_lse so quant-K cells reach
+the fused path instead of asserting.
+
+dca_is_inkernel_liftable() gates the defer to Q8_0/Q4_0/Q4_1/Q5_0/Q5_1/Q6_0 (+ the
+turbo/TCQ families via the existing to_fp16_nc table); f16/bf16 keep the stock path.
+See .wolf/buglog.json bug-720.
+
+diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp
+--- a/llama.cpp/src/dca.cpp
++++ b/llama.cpp/src/dca.cpp
+@@ -204,6 +204,20 @@
+     return ggml_cast(ctx0, x, GGML_TYPE_F16);
+ }
+ 
++// opencoti F5 dca (DCA all-KV C-perf, bug-720): can the FUSED multi-chunk kernel dequant this cache type
++// to f16 INSIDE its launch (single-pass to_fp16_nc into a transient pool buffer, mirroring launch_fattn),
++// instead of dca_lift_to_f16 whole-cache materializing it to f16 in the GRAPH every forward? That graph
++// lift is the prefill cliff: at 98k/q5_0 the two-stage q->f32->f16 cast ran every ubatch and dragged
++// DCA-on prefill to ~138 tps vs ~1212 for the same scalar-quant cache DCA-off (which already converts
++// per-op via launch_fattn). Restrict to the SCALAR quants with a to_fp16_nc converter (convert.cu:843);
++// turbo/TCQ/bf16 keep dca_lift_to_f16 (already a single-pass DIRECT ->f16 cast, not the slow two-stage),
++// f16 is a no-op. The in-op convert is byte-IDENTICAL to the lift (both compute d*q in f32 then round to
++// f16), so this is a pure perf change, gateable by logit-equivalence.
++static bool dca_is_inkernel_liftable(enum ggml_type t) {
++    return t == GGML_TYPE_Q8_0 || t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 ||
++           t == GGML_TYPE_Q5_0 || t == GGML_TYPE_Q5_1 || t == GGML_TYPE_Q6_0;
++}
++
+ ggml_tensor * llm_graph_context::build_attn_dca_core(
+         const llama_kv_cache_context * mctx_cur,
+         llm_graph_input_dca * inp_dca,
+@@ -226,24 +240,53 @@
+     //     the kernel). At f16 this is a no-op (byte-identical to the prior assert path). Context-shift
+     //     stays correct independently: the modular K-shift re-ropes the STORED, still-quantized cache
+     //     via build_rope_shift's existing dequant->rope->requant quantized branch (llama-kv-cache.cpp).
+-    ggml_tensor * k_dca = dca_lift_to_f16(ctx0, mctx_cur->get_k(ctx0, il));
++    ggml_tensor * k_raw = mctx_cur->get_k(ctx0, il);
++    ggml_tensor * v_raw = mctx_cur->get_v(ctx0, il);
++
++    // opencoti F5 dca (DCA all-KV C-perf, bug-720): on the analytical-band (multi-chunk) path the fused
++    // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see
++    // ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case), so pass the quantized cache THROUGH instead of
++    // whole-cache materializing it to f16 via dca_lift_to_f16 every forward (the prefill cliff). Gate:
++    //   - !single_chunk: only the fused op carries the in-launch convert; single_chunk uses stock
++    //     ggml_flash_attn_ext (launch_fattn already converts scalar-quant natively) and runs only while
++    //     n_kv <= chunk (a tiny cache) so its lift cost is negligible — keep it on the safe lift path.
++    //   - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read.
++    //   - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the
++    //     permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift.
++    // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output.
++    const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2];
++    const bool defer_quant_lift =
++        !inp_dca->single_chunk &&
++        dca_is_inkernel_liftable(k_raw->type) &&
++        dca_is_inkernel_liftable(v_raw->type) &&
++        !v_trans_raw;
+ 
+     // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream].
+     //     opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16
+     //     while it is still CONTIGUOUS — a permuted view of a block type is not cleanly cpy-able (the
+     //     block stride spans dim-0), mirroring build_attn_mha's pre-permute turbo cast. f32 V keeps the
+     //     post-permute cast below (unchanged); f16 V is untouched (byte-identical).
+-    ggml_tensor * v = mctx_cur->get_v(ctx0, il);
+-    if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) {
+-        v = dca_lift_to_f16(ctx0, v);
+-    }
+-    const bool v_trans = v->nb[1] > v->nb[2];
+-    ggml_tensor * vp = ggml_permute(ctx0, v, 0, 2, 1, 3);
+-    if (v_trans) {
+-        vp = ggml_transpose(ctx0, vp);
+-    }
+-    if (vp->type == GGML_TYPE_F32) {
+-        vp = ggml_cast(ctx0, vp, GGML_TYPE_F16);
++    ggml_tensor * k_dca;
++    ggml_tensor * vp;
++    if (defer_quant_lift) {
++        // pass scalar-quant K/V straight to the fused op; permute(0,2,1,3) is a clean view (dim-0 = the
++        // block axis, untouched) that the launch's to_fp16_nc dequants with the permuted strides.
++        k_dca = k_raw;
++        vp    = ggml_permute(ctx0, v_raw, 0, 2, 1, 3);
++    } else {
++        k_dca = dca_lift_to_f16(ctx0, k_raw);
++        ggml_tensor * v = v_raw;
++        if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) {
++            v = dca_lift_to_f16(ctx0, v);
++        }
++        const bool v_trans = v->nb[1] > v->nb[2];
++        vp = ggml_permute(ctx0, v, 0, 2, 1, 3);
++        if (v_trans) {
++            vp = ggml_transpose(ctx0, vp);
++        }
++        if (vp->type == GGML_TYPE_F32) {
++            vp = ggml_cast(ctx0, vp, GGML_TYPE_F16);
++        }
+     }
+     ggml_tensor * kp = ggml_permute(ctx0, k_dca, 0, 2, 1, 3);
+ 
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -788,6 +788,17 @@
+     const int64_t n_batch = Q->ne[3];
+     const int64_t n_rows  = n_head * n_q * n_batch;
+     GGML_ASSERT(dst->ne[0] == DV + 1 && "flash_attn_ext_lse dst must pack [O|lse] on ne[0]");
++
++    // opencoti F5 dca (DCA all-KV C-perf, bug-720): the FUSED regime (op_params[4]==3) self-contains its
++    // attention + normalization and now dequant-on-lifts a scalar-quantized K/V INSIDE its own launch
++    // (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case). It never runs the streaming_lse recompute below, so
++    // route it out BEFORE the recompute-only f32-Q / f16-K asserts (the streaming_lse_kernel reads K as f16,
++    // hence those gates). Without this early-out a quantized DCA cache aborts here at line ~792.
++    if (ggml_get_op_params_i32(dst, 4) == 3) {
++        ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst);
++        return;
++    }
++
+     GGML_ASSERT(Q->type == GGML_TYPE_F32 && "flash_attn_ext_lse expects f32 Q for the lse recompute");
+     GGML_ASSERT(K->type == GGML_TYPE_F16 && "flash_attn_ext_lse expects f16 K for the lse recompute");
+ 
+@@ -820,19 +831,11 @@
+     const int dca_chunk  = ggml_get_op_params_i32(dst, 3);
+     const int dca_regime = ggml_get_op_params_i32(dst, 4);
+ 
+-    // opencoti F5 dca perf (T87.pD Branch B): FUSED single-pass DCA. The three band passes (INTER/SUCC/
+-    // INTRA) are folded into ONE block-per-tile kernel that streams K/V once and swaps the pre-roped Q
+-    // variant at the 2 phase boundaries — no per-regime LSE emit, no host combine, no streaming_lse
+-    // fallback. The fused case reads its inputs straight off dst (src[0]=Q_intra, src[1]=K, src[2]=V,
+-    // src[3]=mask_intra, src[4]=pos_q_real, src[5]=Q_succ, src[6]=Q_inter, op_params[3]=chunk) and writes
+-    // normalized O into the leading DV*n_rows of dst's [O|lse] buffer (the trailing lse row is left
+-    // untouched — dca.cpp slices it off). Bands partition the causal range [0,i] ⇒ one online-softmax
+-    // pass is bit-faithful to the 3-op + host LSE combine it replaces.
+-    if (dca_regime == 3) {
+-        ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst);
+-        return;
+-    }
+-
++    // opencoti F5 dca perf (T87.pD Branch B): the FUSED single-pass DCA regime (op_params[4]==3) — three
++    // band passes (INTER/SUCC/INTRA) folded into ONE block-per-tile kernel, no per-regime LSE emit / host
++    // combine / streaming_lse fallback — is dispatched at the TOP of this function (bug-720), before the
++    // f32-Q / f16-K recompute asserts, so a quantized DCA cache reaches its in-launch dequant. Only the
++    // non-fused (regime 0/1/2) lse-recompute path continues below.
+     float * lse_out = (float *) dst->data + DV * n_rows;
+     opencoti_fattn_dst_lse         = lse_out;
+     opencoti_fattn_dst_lse_written = false;
+diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+@@ -2360,7 +2360,14 @@
+     const ggml_tensor * Qi   = dst->src[6]; // Q_inter
+     const ggml_tensor * Ms   = dst->src[7]; // opencoti F5 dca (#617B): dense SUCC mask  (NULL ⇒ fast path)
+     const ggml_tensor * Mi   = dst->src[8]; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ fast path)
+-    GGML_ASSERT(K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && "dca fused: f16 K/V required");
++    // opencoti F5 dca (DCA all-KV C-perf, bug-720): the fused kernel reads f16 K/V, but a scalar-quantized
++    // cache is now dequant'd to f16 IN THIS LAUNCH below (single-pass to_fp16_nc into a transient pool
++    // buffer, mirroring launch_fattn) instead of being whole-cache materialized to f16 in the graph every
++    // forward (the prefill cliff: ~138 vs ~1212 tps at 98k/q5_0). Accept f16, or any type with a to_fp16_nc
++    // converter; reject anything else (the kernel would have no f16 source).
++    GGML_ASSERT((K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) &&
++                (V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) &&
++                "dca fused: K/V must be f16 or have a to_fp16_nc converter");
+     GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors");
+     // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER
+     // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent.
+@@ -2410,6 +2417,44 @@
+     ggml_cuda_pool & pool = ctx.pool();
+     cudaStream_t main_stream = ctx.stream();
+ 
++    // opencoti F5 dca (DCA all-KV C-perf, bug-720): dequant-on-lift a scalar-quantized K/V into a transient
++    // f16 pool buffer here — the SAME single-pass to_fp16_nc convert the stock MMA path uses (launch_fattn,
++    // fattn-common.cuh) — replacing the old two-stage whole-cache graph lift (dca.cpp dca_lift_to_f16,
++    // q->f32->f16) that ran every forward and dominated quant-KV DCA-on prefill. K/V are PERMUTED views
++    // (kp/vp, dim-0 = the contiguous block axis), so use the non-contiguous (strided) converter and pack to
++    // a contiguous f16 slot; the kernel then reads the slot with contiguous strides. f16 K/V skip the
++    // convert (k_data/v_data alias the tensor, strides unchanged ⇒ byte-identical to the prior f16 path).
++    ggml_cuda_pool_alloc<half> K_f16(pool);
++    ggml_cuda_pool_alloc<half> V_f16(pool);
++    const char * k_data = (const char *) K->data;
++    size_t nb_k1 = K->nb[1], nb_k2 = K->nb[2], nb_k3 = K->nb[3];
++    const char * v_data = (const char *) V->data;
++    size_t nb_v1 = V->nb[1], nb_v2 = V->nb[2], nb_v3 = V->nb[3];
++    if (K->type != GGML_TYPE_F16) {
++        const size_t ts = ggml_type_size(K->type);
++        GGML_ASSERT(K->nb[0] == ts && "dca fused: quant K must have contiguous dim-0 (block axis)");
++        to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(K->type);
++        K_f16.alloc(ggml_nelements(K));
++        to_fp16(k_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3],
++                (int64_t) (nb_k1 / ts), (int64_t) (nb_k2 / ts), (int64_t) (nb_k3 / ts), main_stream);
++        k_data = (const char *) K_f16.ptr;
++        nb_k1  = K->ne[0] * sizeof(half);
++        nb_k2  = K->ne[1] * nb_k1;
++        nb_k3  = K->ne[2] * nb_k2;
++    }
++    if (V->type != GGML_TYPE_F16) {
++        const size_t ts = ggml_type_size(V->type);
++        GGML_ASSERT(V->nb[0] == ts && "dca fused: quant V must have contiguous dim-0 (block axis)");
++        to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(V->type);
++        V_f16.alloc(ggml_nelements(V));
++        to_fp16(v_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3],
++                (int64_t) (nb_v1 / ts), (int64_t) (nb_v2 / ts), (int64_t) (nb_v3 / ts), main_stream);
++        v_data = (const char *) V_f16.ptr;
++        nb_v1  = V->ne[0] * sizeof(half);
++        nb_v2  = V->ne[1] * nb_v1;
++        nb_v3  = V->ne[2] * nb_v2;
++    }
++
+     const int ntiles_x     = (Q->ne[1] + ncols1 - 1) / ncols1;
+     const int gqa_ratio    = Q->ne[2] / K->ne[2];
+     const int ntiles_z_gqa = (gqa_ratio + ncols2 - 1) / ncols2;
+@@ -2490,7 +2535,7 @@
+ 
+         const dim3 grid(nblocks, 1, 1);
+         kernel<<<grid, block_dim, nbytes_shared_total, main_stream>>>(
+-            (const char *) Q->data, (const char *) K->data, (const char *) V->data,
++            (const char *) Q->data, k_data, v_data, // opencoti dca bug-720: f16 slot (or aliased f16 tensor)
+             (const char *) mask->data,
+             Ms ? (const char *) Ms->data : nullptr, // opencoti F5 dca (#617B): dense SUCC mask  (null ⇒ fast path)
+             Mi ? (const char *) Mi->data : nullptr, // opencoti F5 dca (#617B): dense INTER mask (null ⇒ fast path)
+@@ -2499,8 +2544,8 @@
+             (float *) KQV->data, dst_meta_ptr,
+             scale, 0.0f /*max_bias*/, 1.0f /*m0*/, 1.0f /*m1*/, n_head_log2, logit_softcap,
+             Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],
+-            K->ne[0], K->ne[1], K->ne[2], K->ne[3], K->nb[1], K->nb[2], K->nb[3],
+-            V->nb[1], V->nb[2], V->nb[3],
++            K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb_k1, nb_k2, nb_k3, // opencoti dca bug-720: slot strides
++            nb_v1, nb_v2, nb_v3,
+             mask->ne[1], mask->ne[2], mask->ne[3], mask->nb[1], mask->nb[2], mask->nb[3]);
+         CUDA_CHECK(cudaGetLastError());
+ 

patches/0086-f16-moe-mmf-prefill.patch

@@ -0,0 +1,49 @@
+From: opencoti
+Subject: [PATCH 0086] F16 MoE prefill — route large-batch f16 mul_mat_id to MMF (#555)
+
+ggml_cuda_mul_mat_id had no fast CUDA path for f16/bf16/f32 experts at prefill batch:
+MMQ is quantized-only, and ggml_cuda_should_use_mmf rejected mul_mat_id whenever
+src1_ncols (= n_tokens) exceeded 512 (or 128 when the expert dim > 1024). At prefill
+(~32k tokens) that gate always failed, so f16 MoE fell into the host-side sorted
+fallback in ggml_cuda_mul_mat_id: a per-call cudaStreamSynchronize (which also disables
+CUDA graphs) + an O(n_expert*n_tokens*n_used) host sort + a per-expert cuBLAS GEMM loop.
+Quantized MoE dodges all of it via the single fused MMQ-id kernel. Net: f16 128e prefill
+ran ~12x slower than Q6_K (690 vs 8857 tok/s on Gemma-4-A4B 128e).
+
+The MMF kernel's `ncols_dst > 16` branch (ggml_cuda_mul_mat_f, this file) already performs
+GPU-side expert compaction (ggml_cuda_launch_mm_ids_helper, mmid.cuh) entirely on-stream
+and handles arbitrary n_tokens -- the gate was a conservative tuning heuristic, not a
+capability limit. This compiles out that mul_mat_id gate so large-batch f16/bf16/f32 MoE
+rides the fast GPU-compacted MMF path. Measured on a real 128e-F16 (bs2 RTX PRO 6000):
+prefill 690 -> 3081 tok/s (4.47x); RULER-vt@32k retrieval 100.0 (== Q6_K, correctness-clean).
+The residual gap to Q6_K is just the legitimate f16-vs-q6 weight-bandwidth ratio (~2.2x).
+Quantized MoE (MMQ path) and dense f16 mul_mat are untouched -- both are rejected/handled
+before this gate. The upstream gate is preserved, compiled out, behind
+-DOPENCOTI_MMF_LEGACY_GATE.
+
+diff --git a/llama.cpp/ggml/src/ggml-cuda/mmf.cu b/llama.cpp/ggml/src/ggml-cuda/mmf.cu
+--- a/llama.cpp/ggml/src/ggml-cuda/mmf.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/mmf.cu
+@@ -160,11 +160,22 @@
+     }
+ 
+     if (mul_mat_id) {
++        // opencoti-hook: f16-moe-prefill (#555) — the upstream heuristic below rejected
++        // large-batch (prefill) f16/bf16/f32 mul_mat_id, sending it to the host-sync sorted
++        // fallback in ggml_cuda_mul_mat_id (per-call cudaStreamSynchronize + per-expert cuBLAS
++        // loop + disabled CUDA graphs => measured 12x slower than the quantized MMQ-id path:
++        // 690 vs 8857 tok/s prefill on Gemma-4-A4B 128e). The MMF kernel's `ncols_dst > 16`
++        // branch already performs GPU-side expert compaction (ggml_cuda_launch_mm_ids_helper,
++        // mmid.cuh) entirely on-stream and handles arbitrary n_tokens, so keep f16 MoE on the
++        // fast fused MMF path for any batch size. Upstream gate preserved (disabled) for ref;
++        // re-enable with -DOPENCOTI_MMF_LEGACY_GATE.
++#ifdef OPENCOTI_MMF_LEGACY_GATE
+         if (src0_ne[1] <= 1024 && src1_ncols > 512) {
+             return false;
+         } else if(src0_ne[1] > 1024 && src1_ncols > 128) {
+             return false;
+         }
++#endif
+     } else {
+         if (GGML_CUDA_CC_IS_RDNA3_0(cc) && src1_ncols > 8) {
+             return false;

patches/0087-dca-yarn-factor.patch

@@ -0,0 +1,56 @@
+From: opencoti
+Subject: [PATCH 0087] DCA: wire --dca-yarn-factor into build_dca_rope mscale (bug-740 / #550)
+
+--dca-yarn-factor was fully plumbed (arg.cpp -> params -> cparams.dca_yarn_factor,
+public C API, TS adapter) but never READ by the rope path, so the flag was inert
+(bug-740). build_dca_rope now composes it into the attention scaling:
+  dca_attn_factor = attn_factor * cparams.dca_yarn_factor
+and passes that to both ggml_rope_multi (MROPE) and ggml_rope_ext (NEOX) in place of
+the bare attn_factor. This is the YaRN mscale piece of the DCA+YaRN composition: DCA
+already keeps q-k distances in-distribution via the per-regime chunk remap, so only the
+softmax temperature (mscale) needs stretching for ctx past the chunk grid -- freq_scale
+/ freq_base stay the model's baked values. Host-only (graph construction); the existing
+CUDA rope kernel reads attn_factor from op_params unchanged, so no DSO rebuild.
+
+Default 1.0 => dca_attn_factor == attn_factor exactly (x1.0f is bit-identity for finite
+floats), so the flag-unset path is byte-identical to upstream -- the fix only makes a
+previously-inert flag live; it cannot regress default behavior.
+
+Empirical characterization (Qwen3-4B, native 40960, DCA chunk 16384, RULER 5-needle
+substring under ignore_eos; NOT greedy-first-token):
+  f16 KV @ 2.4x native: yarn 1.0->5/5  1.25->4/5  1.5->1/5  2.0->0/5  (monotonic: flag live)
+  q8_0 KV ladder yarn=1.0: 2.4x->4  4.0x->5  5.0x->3  6.0x->4  (plain-DCA holds deep)
+  q8_0 KV @ 6.0x native: yarn 1.0->4  1.1->4  1.25->4  (flat; 1.18 = principled mscale, no win)
+Verdict: the mscale lever has NO beneficial regime for DCA retrieval -- it only
+over-sharpens already-concentrated attention. DEFAULT 1.0 IS CORRECT. Ships as a
+power-user knob (raising it degrades retrieval); documented in docs/evaluations/context.md.
+
+diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp
+--- a/llama.cpp/src/dca.cpp
++++ b/llama.cpp/src/dca.cpp
+@@ -164,15 +164,23 @@
+ // Rope the DCA per-regime Q (or insert-K) faithfully for the arch. `pos` must be the width
+ // build_attn_inp_dca allocated: 4*n_tokens for MROPE (section split below), n_tokens for NEOX.
+ ggml_tensor * llm_graph_context::build_dca_rope(ggml_tensor * x, ggml_tensor * pos, const dca_rope & rp) const {
++    // opencoti-hook: dca-yarn-factor (bug-740 / #550) — compose a YaRN mscale stretch with DCA.
++    // cparams.dca_yarn_factor (1.0 = none; set by --dca-yarn-factor) multiplies the attention scaling
++    // (mscale / attn_factor) so the extending-regime softmax is re-tempered for ctx past the chunk grid
++    // (the final 2-4x of the DCA+YaRN sub-native extension recipe; see docs/features/gemma4_dca.md). DCA
++    // already keeps q-k distances in-distribution via the per-regime chunk remap, so only the mscale needs
++    // stretching — freq_scale/freq_base stay the model's baked values. Default 1.0 => this rope call is
++    // byte-identical to upstream (inert unless the flag is set).
++    const float dca_attn_factor = attn_factor * cparams.dca_yarn_factor;
+     if (dca_is_mrope(rope_type)) {
+         int sections[GGML_MROPE_SECTIONS] = { rp.sections[0], rp.sections[1], rp.sections[2], rp.sections[3] };
+         return ggml_rope_multi(ctx0, x, pos, rp.freq_factors,
+                 rp.n_rot, sections, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale,
+-                ext_factor, attn_factor, beta_fast, beta_slow);
++                ext_factor, dca_attn_factor, beta_fast, beta_slow);
+     }
+     return ggml_rope_ext(ctx0, x, pos, rp.freq_factors,
+             rp.n_rot, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale,
+-            ext_factor, attn_factor, beta_fast, beta_slow);
++            ext_factor, dca_attn_factor, beta_fast, beta_slow);
+ }
+ 
+ // opencoti F5 dca #444 (DCA all-KV C1): dequant-on-lift a cached K/V tensor to f16 for the f16-only

patches/0088-sparsev-autopolicy.patch

@@ -0,0 +1,63 @@
+From: opencoti
+Subject: [PATCH 0088] sparse-v auto-policy: self-configure quantized-V skip from model arch (#565)
+
+The FA-VEC quantized-V skip (sparse-V, #546) is a MEASURED lossless decode win on
+interleaved-SWA models (Gemma-4: the small, concentrated global V-cache lets a
+peak-relative static tau=0.05 harvest the sub-mean V tail — RULER niah 100 at
+64k/128k/256k on A4B-62e AND official Gemma-4-31B), but a no-win on full-attention
+models (Qwen: diffuse weights cluster near the mean, nothing safely skippable). The
+kernel reads its threshold from the TURBO_SPARSE_V_TAU env (ggml-cuda is a runtime
+DSO, so env is the cross-DSO channel). Until now a user had to know to set that env.
+
+This puts the intelligence in the BINARY so a STANDALONE `llamafile --server` self-
+configures: at llama_context construction, when the user has NOT set TURBO_SPARSE_V_TAU
+and the model is iSWA (hparams.swa_type != NONE) with a quantized V cache, default the
+env to 0.05 (overwrite=0 — an explicit env still wins). Full-attention models leave it
+unset -> kernel tau=0 -> dense, byte-identical. f16/bf16 V is inert via the kernel's own
+`if constexpr type_V != F16/BF16` guard. Host-only (no DSO change); the existing kernel
+getenv reads the pre-populated value. Announced once at WARN (the llamafile server
+suppresses the model-load INFO block, and a binary that auto-changes inference should
+say so + tell the user how to override).
+
+Verified (3090): decision table PASS (policy fires iff iSWA+quant-V+no-env: A4B q8_0
+auto=engaged, override/f16/Qwen-full-attn=off) + A/B at 24k lossless + non-slower.
+---
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+index 722bd7f..2d87414 100644
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -54,6 +54,34 @@ llama_context::llama_context(
+ 
+     const auto & hparams = model.hparams;
+ 
++    // opencoti-hook: sparse-v auto-policy (#565) — see docs/features/sparse_attn.md
++    // Self-configure the FA-VEC quantized-V skip (#546) from the model's OWN architecture, so a
++    // STANDALONE `llamafile --server` gets the win with no flags (the intelligence lives in the binary,
++    // not the opencoti TS adapter). The skip is a MEASURED lossless decode win on interleaved-SWA models
++    // (Gemma-4: the tiny, concentrated global V-cache lets a peak-relative static τ=0.05 harvest the
++    // sub-mean V tail — niah 100 at 64k/128k/256k on A4B-62e AND official Gemma-4-31B), but a no-win on
++    // full-attention models (Qwen: diffuse weights cluster near the mean, nothing safely skippable). It
++    // only engages for a QUANTIZED V cache (the kernel's `if constexpr type_V != F16/BF16` guard; -ctv
++    // f16 is inert). ggml-cuda is a runtime-loaded DSO, so the kernel reads its threshold from the
++    // TURBO_SPARSE_V_TAU env (the existing cross-DSO channel) — we just pre-populate a smart default when
++    // the user has not, with overwrite=0 so an explicit TURBO_SPARSE_V_TAU (or eps/sink/recent) still
++    // wins. Set as early as possible (before any graph_reserve / decode) so the kernel's lazy getenv
++    // caches the policy value. Full-attention models leave the env unset → kernel τ=0 → dense, byte-identical.
++    if (!hparams.vocab_only && getenv("TURBO_SPARSE_V_TAU") == nullptr) {
++        const bool is_iswa = hparams.swa_type != LLAMA_SWA_TYPE_NONE;
++        // Mirror the kernel's engage guard exactly: any non-float V cache (q8_0/q4_0/turbo*/tcq) is eligible.
++        const bool quant_v = params.type_v != GGML_TYPE_F16 && params.type_v != GGML_TYPE_BF16 &&
++                             params.type_v != GGML_TYPE_F32;
++        if (is_iswa && quant_v) {
++            setenv("TURBO_SPARSE_V_TAU", "0.05", /*overwrite=*/0);
++            // WARN level (not INFO): the llamafile server suppresses the model-load INFO block, and a binary
++            // that auto-changes inference behavior should announce it audibly + tell the user how to override.
++            LLAMA_LOG_WARN("%s: sparse-v auto-policy: iSWA + quantized V (%s) -> TURBO_SPARSE_V_TAU=0.05 "
++                           "(lossless decode win; export TURBO_SPARSE_V_TAU to override)\n",
++                    __func__, ggml_type_name(params.type_v));
++        }
++    }
++
+     cparams.n_seq_max = std::max(1u, params.n_seq_max);
+     if (cparams.n_seq_max > LLAMA_MAX_SEQ) {
+         throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ));

patches/0090-rolling-kv-compute-reserve.patch

@@ -0,0 +1,333 @@
+From: opencoti
+Subject: [PATCH 0089] rolling-KV two-pass compute-buffer reserve (bug-1342)
+
+The POSITION_WINDOW resident-window sizer (opencoti_compute_resident_window_cells)
+held back a FIXED OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES = 1536 MiB for the compute
+scratch that graph_reserve allocates AFTER the KV cache. But that compute buffer
+SCALES with context — measured 3.63 GiB (3 800 045 568 B) at 262 144-524 288 ctx on
+an RTX PRO 6000 — so the sizer kept ~2.1 GiB too much KV resident and graph_reserve
+OOM'd at boot:
+
+    ggml_gallocr_reserve_n_impl: failed to allocate CUDA0 buffer of size 3800045568
+    graph_reserve: failed to allocate compute buffers
+
+Fix: measure the real compute buffer instead of guessing it. Two-pass context init,
+gated to the GPU flash-attn residency-managed path so the common resident case stays
+single-pass / byte-identical:
+
+  Pass 1 (measure): create_memory with kv_window_measure_pass=true -> the window sizer
+    returns a MINIMAL resident window (tiny device KV, boots for ANY compute size).
+    sched_reserve() runs the pp worst-case reserve; sum the GPU backends' compute
+    buffer via backend_buf_exp_size.
+  Pass 2 (real): re-create_memory with kv_compute_reserve_mib = measured + 256 MiB
+    safety; the sizer now holds back the true compute buffer and sizes the window
+    against (budget - model - measured). sched_need_reserve=true; sched_reserve() again.
+
+The compute buffer is independent of the window/tail split (it depends on
+n_ubatch x n_kv x n_embd, and n_kv = full ctx regardless of the resident split), so
+one measurement suffices. kv_compute_reserve_mib / kv_window_measure_pass are INTERNAL
+cparams (set by the context two-pass, not from llama_context_params); hybrid-memory
+paths pass 0/false (two-pass disabled). Host-only change — see
+docs/features/rolling_kv_compute_reserve.md.
+
+diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h
+--- a/llama.cpp/src/llama-cparams.h
++++ b/llama.cpp/src/llama-cparams.h
+@@ -35,6 +35,16 @@
+     // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob.
+     // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head, 2 = window.
+     uint32_t kv_residency_mode;
++    // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) —
++    // see docs/features/rolling_kv_compute_reserve.md. INTERNAL (not user params):
++    // set by the llama_context two-pass measurement, NOT from llama_context_params.
++    // kv_compute_reserve_mib = MEASURED GPU compute-buffer size to hold back from the
++    // resident-window budget (0 = use the fixed OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES
++    // default, i.e. pass 1 / two-pass disabled). kv_window_measure_pass forces the
++    // window sizer to a minimal resident window (tiny device KV, boots for ANY compute
++    // size) so pass 1 can measure the real compute buffer without OOM.
++    uint32_t kv_compute_reserve_mib;
++    bool     kv_window_measure_pass;
+     // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention.
+     // 0 = off (M2 concat path runs; binary equivalent to upstream when
+     //         headinfer_gpu_heads_frac == 1.0).
+diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -106,6 +106,12 @@
+     cparams.pcie_bw_gbps = params.pcie_bw_gbps;
+     // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md
+     cparams.kv_residency_mode = params.kv_residency_mode;
++    // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). INTERNAL:
++    // the memory init below runs a measurement pass, then re-creates with the measured
++    // reserve; these start at the "pass 1 / default reserve" values. See
++    // docs/features/rolling_kv_compute_reserve.md.
++    cparams.kv_compute_reserve_mib = 0;
++    cparams.kv_window_measure_pass = false;
+     // opencoti F5 M3 neo — see docs/features/advanced_kv.md
+     cparams.neo_pipeline_mode = params.neo_pipeline_mode;
+     // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md
+@@ -373,6 +379,18 @@
+         }
+     }
+ 
++    // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) —
++    // see docs/features/rolling_kv_compute_reserve.md. The resident-window sizer must
++    // hold back the compute buffer (allocated AFTER the KV by sched_reserve), but its
++    // size scales with context and is only known post-reserve. So: PASS 1 creates a
++    // MINIMAL resident window (tiny device KV, boots for any compute size), sched_reserve
++    // measures the real GPU compute buffer, then PASS 2 re-creates the KV sized against
++    // (free - model - measured). Scoped to the GPU flash-attn residency-managed path so
++    // the common resident case is unaffected (the pass-2 result is identical there).
++    const bool opencoti_two_pass = !hparams.vocab_only
++            && cparams.flash_attn && cparams.offload_kqv
++            && !model.devices.empty() && cparams.kv_residency_mode != 1;
++
+     // init the memory module
+     if (!hparams.vocab_only) {
+         llama_memory_params params_mem = {
+@@ -382,6 +400,8 @@
+             /*.ctx_type= */ cparams.ctx_type,
+         };
+ 
++        // opencoti bug-1342: pass 1 forces a minimal resident window (measure mode).
++        cparams.kv_window_measure_pass = opencoti_two_pass;
+         memory.reset(model.create_memory(params_mem, cparams));
+     }
+ 
+@@ -450,6 +470,41 @@
+ 
+         sched_reserve();
+ 
++        // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). Pass 1
++        // (above) sized a minimal resident window so sched_reserve could not OOM; the real
++        // GPU compute buffer is now in backend_buf_exp_size. Sum it over the GPU backends,
++        // re-create the KV with that measured reserve, and re-reserve. The compute buffer is
++        // independent of the window/tail split, so the measurement is exact for pass 2. See
++        // docs/features/rolling_kv_compute_reserve.md.
++        if (opencoti_two_pass && !model.hparams.no_alloc) {
++            size_t gpu_compute = 0;
++            for (size_t i = 0; i < backend_ptrs.size(); ++i) {
++                if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) == GGML_BACKEND_DEVICE_TYPE_GPU) {
++                    gpu_compute += backend_buf_exp_size[i];
++                }
++            }
++            // + 256 MiB safety margin over the measured worst-case pp compute buffer.
++            const uint32_t measured_mib = (uint32_t) (gpu_compute / (1024 * 1024)) + 256;
++            LLAMA_LOG_INFO("%s: rolling-kv two-pass — measured GPU compute buffer = %u MiB; "
++                    "re-sizing resident window against it\n", __func__, measured_mib);
++            cparams.kv_compute_reserve_mib = measured_mib;
++            cparams.kv_window_measure_pass = false;
++
++            // free the pass-1 KV (incl. its host tail) BEFORE creating pass 2 to avoid a
++            // transient double host-tail allocation.
++            memory.reset();
++            llama_memory_params params_mem2 = {
++                /*.type_k   =*/ params.type_k,
++                /*.type_v   =*/ params.type_v,
++                /*.swa_full =*/ params.swa_full,
++                /*.ctx_type= */ cparams.ctx_type,
++            };
++            memory.reset(model.create_memory(params_mem2, cparams));
++
++            sched_need_reserve = true;
++            sched_reserve();
++        }
++
+         if (!cparams.flash_attn) {
+             if (ggml_is_quantized(params.type_v)) {
+                 throw std::runtime_error("quantized V cache was requested, but this requires Flash Attention");
+diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h
+--- a/llama.cpp/src/llama-kv-cache.h
++++ b/llama.cpp/src/llama-kv-cache.h
+@@ -131,6 +131,15 @@
+                      // 0 = auto (POSITION_WINDOW when eligible, else M2 head split),
+                      // 1 = head (force M2 split), 2 = window (force POSITION_WINDOW).
+                      uint32_t   kv_residency_mode,
++                     // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve
++                     // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md.
++                     // compute_reserve_mib: MEASURED GPU compute-buffer size to hold
++                     // back from the resident-window budget (0 = fixed default).
++                     // window_measure_pass: force a MINIMAL resident window (tiny
++                     // device KV) so the context two-pass can measure the real
++                     // compute buffer without OOM.
++                     uint32_t   compute_reserve_mib,
++                     bool       window_measure_pass,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_pad,
+                      uint32_t   n_swa,
+diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -138,6 +138,7 @@
+         uint32_t kv_size,
+         uint32_t n_stream,
+         uint32_t vram_target_mib,
++        size_t   compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back
+         const std::function<bool(int32_t)> & filter) {
+     // No GPU offload → nothing is "GPU-resident" and nothing to spill; the M2
+     // split is only meaningful for offloaded layers. Stay at vanilla.
+@@ -187,9 +188,10 @@
+     if (vram_target_mib > 0) {
+         budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024);
+     }
+-    budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES
+-        ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES
+-        : 0;
++    // opencoti bug-1342: hold back the MEASURED compute buffer (two-pass) instead of the
++    // fixed 1536 MiB guess when the context supplied it; 0 → fall back to the default.
++    const size_t reserve = compute_reserve_bytes ? compute_reserve_bytes : OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES;
++    budget = budget > reserve ? budget - reserve : 0;
+ 
+     if (kv_demand <= budget) {
+         LLAMA_LOG_INFO("%s: auto residency = GPU_RESIDENT (KV %.0f MiB fits budget "
+@@ -235,10 +237,21 @@
+         uint32_t n_stream,
+         uint32_t vram_target_mib,
+         uint32_t n_pad,
++        size_t   compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back
++        bool     measure_pass,          // opencoti bug-1342: pass 1 → minimal window (tiny device KV)
+         const std::function<bool(int32_t)> & filter) {
+     if (!offload || kv_size == 0) {
+         return kv_size; // no position-axis spill without GPU offload
+     }
++    // opencoti bug-1342 two-pass: the MEASUREMENT pass forces a MINIMAL resident window
++    // (one FA stride, ~0.5 MiB device KV) so pass 1 boots for ANY compute-buffer size.
++    // The context then measures the real compute buffer and re-sizes in pass 2. Aligned
++    // to lcm(n_pad, FATTN_KQ_STRIDE=256) so the tiny window is a valid FA tiling.
++    if (measure_pass) {
++        const uint32_t pad0    = n_pad ? n_pad : 32;
++        const uint32_t walign0 = pad0 >= 256 ? pad0 : 256;
++        return kv_size > walign0 ? walign0 : kv_size;
++    }
+ 
+     size_t per_cell = 0; // bytes for ONE position cell across all this cache's KV layers
+     ggml_backend_dev_t dev = nullptr;
+@@ -327,6 +340,8 @@
+                  uint32_t   vram_target_mib,
+                  float      pcie_bw_gbps,
+                  uint32_t   kv_residency_mode,
++                 uint32_t   compute_reserve_mib,
++                 bool       window_measure_pass,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_pad,
+                  uint32_t   n_swa,
+@@ -383,9 +398,14 @@
+     // A non-negative frac is the manual escape hatch / upstream default, used
+     // verbatim. Every split decision below reads eff_gpu_heads_frac, not the raw
+     // ctor argument, so the sentinel never leaks into the allocation math.
++    // opencoti bug-1342: the compute-buffer reserve to hold back from the resident budget.
++    // compute_reserve_mib > 0 = the MEASURED GPU compute buffer (context two-pass, pass 2);
++    // 0 = pass 1 / two-pass disabled → the sizers fall back to the fixed default.
++    const size_t opencoti_compute_reserve_bytes = (size_t) compute_reserve_mib * 1024 * 1024;
+     const float eff_gpu_heads_frac = headinfer_gpu_heads_frac < 0.0f
+         ? opencoti_compute_auto_gpu_heads_frac(model, hparams, type_k, type_v,
+-                v_trans, offload, kv_size, n_stream, vram_target_mib, filter)
++                v_trans, offload, kv_size, n_stream, vram_target_mib,
++                opencoti_compute_reserve_bytes, filter)
+         : headinfer_gpu_heads_frac;
+ 
+     // opencoti F5 M7 Stage 3c — position-axis window sizing. Computed + logged now so the
+@@ -395,7 +415,7 @@
+     // value is intentionally unused downstream at this sub-stage. See docs/features/advanced_kv.md.
+     const uint32_t window_cells = opencoti_compute_resident_window_cells(
+             model, hparams, type_k, type_v, v_trans, offload, kv_size, n_stream,
+-            vram_target_mib, n_pad, filter);
++            vram_target_mib, n_pad, opencoti_compute_reserve_bytes, window_measure_pass, filter);
+     // opencoti F5 M7 Stage 4 (#338) — position-window mode is driven by the
+     // --kv-residency-mode knob (cparams.kv_residency_mode): 0=auto, 1=head, 2=window.
+     // The ELIGIBILITY predicate is the validated POSITION_WINDOW class: the sizer found a
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h
+--- a/llama.cpp/src/llama-kv-cache-iswa.h
++++ b/llama.cpp/src/llama-kv-cache-iswa.h
+@@ -41,6 +41,12 @@
+                      // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob;
+                      // forwarded to both kv_base and kv_swa. 0 = auto, 1 = head, 2 = window.
+                      uint32_t   kv_residency_mode,
++                     // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve
++                     // (bug-1342); forwarded to both kv_base and kv_swa (only the
++                     // overflowing base cache windows). See
++                     // docs/features/rolling_kv_compute_reserve.md.
++                     uint32_t   compute_reserve_mib,
++                     bool       window_measure_pass,
+                      uint32_t   n_seq_max,
+                      uint32_t   n_ubatch,
+                      uint32_t   n_pad,
+diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp
+--- a/llama.cpp/src/llama-kv-cache-iswa.cpp
++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp
+@@ -26,6 +26,8 @@
+                  uint32_t   vram_target_mib,
+                  float      pcie_bw_gbps,
+                  uint32_t   kv_residency_mode,
++                 uint32_t   compute_reserve_mib,
++                 bool       window_measure_pass,
+                  uint32_t   n_seq_max,
+                  uint32_t   n_ubatch,
+                  uint32_t   n_pad,
+@@ -70,6 +72,7 @@
+             model, type_k, type_v,
+             v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms,
+             headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode,
++            compute_reserve_mib, window_measure_pass,
+             n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, sparse_attn_block_size);
+ 
+     LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
+@@ -78,6 +81,7 @@
+             model, type_k, type_v,
+             v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms,
+             headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode,
++            compute_reserve_mib, window_measure_pass,
+             n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, sparse_attn_block_size);
+ }
+ 
+diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp
+--- a/llama.cpp/src/llama-model.cpp
++++ b/llama.cpp/src/llama-model.cpp
+@@ -2100,6 +2100,10 @@
+                                 cparams.pcie_bw_gbps,
+                                 // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob
+                                 cparams.kv_residency_mode,
++                                // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve
++                                // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md
++                                cparams.kv_compute_reserve_mib,
++                                cparams.kv_window_measure_pass,
+                                 cparams.n_seq_max,
+                                 cparams.n_ubatch,
+                                 1,
+@@ -2130,6 +2134,10 @@
+                                 cparams.pcie_bw_gbps,
+                                 // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob
+                                 cparams.kv_residency_mode,
++                                // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve
++                                // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md
++                                cparams.kv_compute_reserve_mib,
++                                cparams.kv_window_measure_pass,
+                                 cparams.n_seq_max,
+                                 1,
+                                 hparams.n_swa,
+diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp
+--- a/llama.cpp/src/llama-memory-hybrid.cpp
++++ b/llama.cpp/src/llama-memory-hybrid.cpp
+@@ -55,6 +55,9 @@
+         0.0f,
+         // opencoti F5 M7 Stage 4 — hybrid memory does not thread the residency-mode knob; 0 = auto.
+         0,
++        // opencoti F5 M7 bug-1342 — hybrid memory does not use the window two-pass; 0 / false.
++        0,
++        false,
+         n_seq_max,
+         n_pad,
+         n_swa,
+diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp
++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp
+@@ -56,6 +56,9 @@
+         0.0f,
+         // opencoti F5 M7 Stage 4 — hybrid-iswa does not thread the residency-mode knob; 0 = auto.
+         0,
++        // opencoti F5 M7 bug-1342 — hybrid-iswa does not use the window two-pass; 0 / false.
++        0,
++        false,
+         n_seq_max,
+         n_ubatch,
+         n_pad,

patches/0091-multi-session-graph-nodes.patch

@@ -0,0 +1,50 @@
+From: opencoti
+Subject: [PATCH 0090] multi-session graph-node budget for --parallel >=2 (bug-2094)
+
+graph_max_nodes() sized the compute-graph meta pool as a single-stream budget
+(8*n_tensors for dense archs; max(n_tokens*40, 32*n_tensors) for the qwen3next/
+qwen35/kimi linear-attn branch), INDEPENDENT of the stream count. But with a
+non-unified KV cache (the default; --parallel N => kv_unified=false) the graph
+builds per-stream KV views/masks/copies (the `for s < n_stream` loops throughout
+llama-kv-cache.cpp), so the actual node count scales with the number of streams.
+On a 48-layer model (Qwen2.5-14B-Instruct-1M) --parallel 2 therefore overflowed
+the pool at graph build:
+
+    llama.cpp/ggml/src/ggml.c:1790: not enough space in the context's memory pool
+
+(server aborts at boot; --parallel 1 was fine; independent of --kv-residency-mode
+and of ctx). Fix: scale the node budget by n_stream = (kv_unified ? 1 : n_seq_max),
+applied to both branches. Tied to the exact per-stream-KV-graph mechanism; a
+UNIFIED cache keeps n_stream=1 -> byte-identical to upstream, so the factor
+self-disables when there is no per-stream graph.
+
+Measured actual n_nodes (RTX 3090, ctx 8192): dense Qwen3-8B 1267/2131/3571 at
+--parallel 1/2/4 (~776 nodes/stream); iSWA Gemma-4-A4B 2647/3367/4567 (~660/stream);
+qwen35 Qwen3.5-9B 1856/2048/2368 (~170/stream). All boot with 2.5-4x headroom.
+Verified on the original repro (bs2 RTX PRO 6000, 14B-1M-Q8_0 --parallel 2): CRASH
+-> BOOT. Host-only (build:llamafile:make); no CUDA DSO, no restamp.
+
+--- a/llama.cpp/src/llama-context.cpp
++++ b/llama.cpp/src/llama-context.cpp
+@@ -2585,10 +2585,19 @@
+ //
+ 
+ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
++    // opencoti-hook: bug-2094 multi-session graph-node budget — see docs/features/rolling_kv_compute_reserve.md
++    // With a non-unified KV cache (--parallel N ⇒ kv_unified=false) the graph builds
++    // per-stream KV views/masks/copies (the `for s < n_stream` loops throughout
++    // llama-kv-cache.cpp), so the node count scales with the stream count. The upstream
++    // budgets below are single-stream; without this factor --parallel ≥2 overflows the
++    // graph-meta pool (ggml.c:1790 "not enough space in the context's memory pool"). A
++    // UNIFIED cache keeps n_stream=1 → byte-identical to upstream, so the factor
++    // self-disables exactly when there is no per-stream graph.
++    const uint32_t n_stream = cparams.kv_unified ? 1u : std::max<uint32_t>(1u, cparams.n_seq_max);
+     if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) {
+-        return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
++        return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors() * n_stream);
+     }
+-    uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors());
++    uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors()) * n_stream;
+     for (const auto & lora : model.loras) {
+         res += lora->get_n_nodes();
+     }

patches/0092-cuda-build-ccache.patch

@@ -0,0 +1,71 @@
+diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh
+index 4781516..5b1c26c 100644
+--- a/llamafile/build-functions.sh
++++ b/llamafile/build-functions.sh
+@@ -238,8 +238,15 @@ compile_gpu_sources_parallel() {
+             obj="$build_dir/${base}.o"
+         fi
+ 
+-        # Skip if object file is newer than source
+-        if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then
++        # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness. The CUDA kernels
++        # live in .cuh HEADERS; template-instance .cu stubs never change, so comparing the object
++        # only to its .cu left STALE objects on header edits → stale DSO. Also require the object
++        # to be newer than the newest ggml-tree header ($OPENCOTI_CUDA_DEPS_NEWEST, exported by
++        # cuda.sh). Compilation runs through ccache (see cuda.sh), so a header edit then recompiles
++        # only the TUs whose expanded includes actually changed. The `-z` guard preserves the old
++        # .cu-only behaviour for any caller that doesn't export the var.
++        if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \
++           && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then
+             echo "[$count/$total] Skipping: $base.cu (up to date)"
+             continue
+         fi
+@@ -298,8 +305,11 @@ compile_ggml_core() {
+         local name="${base%.*}"
+         local obj="$build_dir/ggml-core-${name}.o"
+ 
+-        # Skip if object file is newer than source
+-        if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then
++        # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness (see cuda.sh). Skip
++        # only when the object is newer than the source AND every ggml-tree header, so a header
++        # edit correctly forces a recompile. `-z` guard keeps the old behaviour if the var is unset.
++        if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \
++           && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then
+             echo "  Skipping: $base (up to date)"
+             continue
+         fi
+diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
+index 18d4357..8e38209 100755
+--- a/llamafile/cuda.sh
++++ b/llamafile/cuda.sh
+@@ -293,10 +293,29 @@ collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_Q
+ echo "  Sources: $NUM_SOURCES .cu files"
+ echo ""
+ 
++# opencoti-hook: cuda-build-ccache (bug-858) — see docs/protocols/LLAMAFILE_UPGRADE.md.
++# The CUDA kernels live in .cuh HEADERS (fattn-vec.cuh, fattn-mma-f16.cuh, ...); the
++# template-instance .cu stubs that #include them essentially never change. The per-TU
++# incremental skip in build-functions.sh compared each object ONLY to its .cu file, so editing a
++# .cuh recompiled NOTHING → a silently STALE DSO (bug-858; same class as the host bug-167 -MMD
++# fix, which was never applied to this hand-rolled CUDA loop). Fix: (1) route every TU compile
++# through ccache, whose include-manifest invalidates the cached object when the .cu OR any
++# included header changes; (2) export the newest header mtime in the ggml tree so the no-ccache
++# fallback skip is header-aware. Linking (link_shared_library) stays on raw $NVCC — ccache caches
++# compiles only.
++CUDA_CC="$NVCC"
++if command -v ccache >/dev/null 2>&1; then
++    CUDA_CC="ccache $NVCC"
++    echo "  ccache: ON for CUDA compilation (header-aware via include manifest)"
++else
++    echo "  ccache: NOT FOUND — install ccache for fast + correct incremental CUDA rebuilds"
++fi
++export OPENCOTI_CUDA_DEPS_NEWEST=$(find "$GGML_CUDA_DIR/../.." \( -name '*.cuh' -o -name '*.h' -o -name '*.hpp' \) -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)
++
+ START_TIME=$(date +%s)
+ 
+ # Compile GPU sources
+-compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF"
++compile_gpu_sources_parallel "$CUDA_CC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF"
+ 
+ COMPILE_TIME=$(date +%s)
+ echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds"

patches/0094-tinyblas-small-gemm.patch

@@ -0,0 +1,31 @@
+From: opencoti
+Subject: [PATCH 0094] tinyblas small-GEMM second config (bug-2104, #614)
+
+opencoti-hook: tinyblas-small-gemm — skinny f32 GEMMs (Gemma MoE router
+ffn_gate_inp [2816x128], qwen35 DeltaNet chunk GEMMs) hit a single 128x64-tile
+config = 8 blocks on a 142-SM RTX 6000 (5% occupancy, 274us). Add a second
+<32,32,64,...> config selected when the big config yields <24 blocks (90.7us;
+prefill +25-35% on MoE models).
+
+diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
+index 758f68a..76ba099 100644
+--- a/llamafile/tinyblas.cu
++++ b/llamafile/tinyblas.cu
+@@ -539,6 +539,17 @@ tinyblasStatus_t tinyblasGE_launch(tinyblasHandle_t handle, tinyblasOperation_t
+                                    int ldc) {
+     if (can_use_matvec(aT, bT, m, n, k, alpha, beta))
+         return matvec_launch<WORD>(handle, m, k, A, lda, B, C);
++    // opencoti-hook: tinyblas-small-gemm (#614/bug-2104) — the single 128x64-tile config
++    // launches only ceil(m/64)*ceil(n/128) blocks after the row-major swap below; a skinny
++    // GEMM like the Gemma MoE router ([2816x128] x [2816x512] per layer per ubatch) gets 8
++    // blocks on a 142-SM GPU (~6% occupancy, ~274 us vs cuBLAS ~8 us) and was 35% of A4B
++    // prefill GPU time. When the big config cannot fill the GPU, use a 32x32-tile config
++    // (same deterministic FFMA warp2d kernel, 4x1 thread tiles, 2 warps) so the block
++    // count grows 8x. Threshold 24 keeps every launch that already saturates on the big
++    // tiles untouched.
++    if ((long long)CEIL_DIV(m, 64) * CEIL_DIV(n, 128) < 24)
++        return tinyblasGE_launcher<32, 32, 64, 8, 32, 16, 1, 4, 4, 64>(
++            handle, bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);
+     constexpr int TT = 256;
+     constexpr int BM = 128;
+     constexpr int BN = 64;

patches/0095-prefill-checkpoint-align.patch

@@ -0,0 +1,145 @@
+From: opencoti
+Subject: [PATCH 0095] prefill checkpoint alignment: min-step 8192 + state-buffer recycle pool (bug-2105/2107, #614)
+
+- checkpoint_min_step 256 -> 8192 (align with b9859; stale port default caused
+  several SWA/recurrent state checkpoints per request instead of <=1).
+- opencoti-hook: ckpt-buf-noinit — common_state_buf type + server-side recycle
+  pool (ckpt_buf_pool, cap 4) so steady-state requests reuse warm pages: no
+  GB-class value-init memset and no page-fault storm inside the state D2H copy.
+  (Measured dead end: a non-zeroing allocator alone regressed all models 8-13%.)
+
+diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h
+index b688492..3bb6aa2 100644
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -670,7 +673,7 @@ struct common_params {
+     bool    cache_prompt        = true;  // whether to enable prompt caching
+     bool    cache_idle_slots    = true;  // save and clear idle slots upon starting a new task
+     int32_t n_ctx_checkpoints   = 32;    // max number of context checkpoints per slot
+-    int32_t checkpoint_min_step = 256;   // minimum spacing between context checkpoints
++    int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
+     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
+ 
+     std::string hostname      = "127.0.0.1";
+@@ -1124,14 +1127,21 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
+ // prompt utils
+ //
+ 
++// opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — named buffer type for state snapshots so
++// the server can recycle them across requests (warm pages: no memset, no page-fault storm).
++// Measured dead end kept for the record: a non-zeroing allocator here made the COLD path
++// slower (page faults land inside the state D2H copy loop instead of a fast sequential
++// memset prefault) — plain vector cold + recycled warm is the optimum.
++using common_state_buf = std::vector<uint8_t>;
++
+ struct common_prompt_checkpoint {
+     int64_t n_tokens;
+ 
+     llama_pos pos_min;
+     llama_pos pos_max;
+ 
+-    std::vector<uint8_t> data_tgt;
+-    std::vector<uint8_t> data_dft;
++    common_state_buf data_tgt;
++    common_state_buf data_dft;
+ 
+     size_t size() const;
+ 
+diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp
+index 7bad046..34b50b1 100644
+--- a/llama.cpp/tools/server/server-context.cpp
++++ b/llama.cpp/tools/server/server-context.cpp
+@@ -2119,20 +2158,50 @@ private:
+         return true;
+     }
+ 
++    // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — recycle retired checkpoint state
++    // buffers so the pages stay mapped and warm; a fresh GB-class allocation per request
++    // pays a page-fault storm inside the state D2H copy (worse than the memset it replaced)
++    std::vector<common_state_buf> ckpt_buf_pool;
++
++    void ckpt_buf_recycle(common_prompt_checkpoint & ckpt) {
++        if (ckpt_buf_pool.size() < 4 && ckpt.data_tgt.capacity() > 0) {
++            ckpt_buf_pool.push_back(std::move(ckpt.data_tgt));
++        }
++        if (ckpt_buf_pool.size() < 4 && ckpt.data_dft.capacity() > 0) {
++            ckpt_buf_pool.push_back(std::move(ckpt.data_dft));
++        }
++    }
++
++    common_state_buf ckpt_buf_acquire() {
++        if (ckpt_buf_pool.empty()) {
++            return {};
++        }
++        common_state_buf buf = std::move(ckpt_buf_pool.back());
++        ckpt_buf_pool.pop_back();
++        return buf;
++    }
++
+     // n_tokens_cur: the number of tokens added to the batch for the current slot
+     void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) {
+         while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) {
+             // make room for the new checkpoint, if needed
+-            const auto & cur = slot.prompt.checkpoints.front();
++            auto & cur = slot.prompt.checkpoints.front();
+ 
+             SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
+                     cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024);
+ 
++            ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107)
+             slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin());
+         }
+ 
+         auto & cur = slot.prompt.checkpoints.emplace_back();
+ 
++        // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — seed with warm recycled buffers
++        cur.data_tgt = ckpt_buf_acquire();
++        if (ctx_dft) {
++            cur.data_dft = ckpt_buf_acquire();
++        }
++
+         cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max);
+ 
+         cur.update_tgt(ctx_tgt,       slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+@@ -2980,9 +3042,10 @@ private:
+                             {
+                                 // erase any checkpoints with pos_max > pos_next
+                                 for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) {
+-                                    const auto & cur = *it;
++                                    auto & cur = *it;
+                                     if (cur.pos_max > pos_next) {
+                                         SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024);
++                                        ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107)
+                                         it = slot.prompt.checkpoints.erase(it);
+                                     } else {
+                                         ++it;
+diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp
+index 803e31f..70ca8dc 100644
+--- a/llama.cpp/tools/server/server-task.cpp
++++ b/llama.cpp/tools/server/server-task.cpp
+@@ -2025,8 +2025,8 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t
+         }
+     }
+ 
+-    std::vector<uint8_t> state_data_tgt;
+-    std::vector<uint8_t> state_data_dft;
++    common_state_buf state_data_tgt;   // opencoti-hook: ckpt-buf-noinit (#614/bug-2107)
++    common_state_buf state_data_dft;
+ 
+     // check if we can allocate enough memory for the new state
+     try {
+diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h
+index 933bd99..380dc27 100644
+--- a/llama.cpp/tools/server/server-task.h
++++ b/llama.cpp/tools/server/server-task.h
+@@ -582,8 +582,8 @@ struct server_task_result_apply_lora : server_task_result {
+ };
+ 
+ struct server_prompt_data {
+-    std::vector<uint8_t> main;
+-    std::vector<uint8_t> drft;
++    common_state_buf main;   // opencoti-hook: ckpt-buf-noinit (#614/bug-2107)
++    common_state_buf drft;
+ 
+     size_t size() const {
+         return main.size() + drft.size();

patches/0097-native-elf-cmake.patch

@@ -0,0 +1,140 @@
+opencoti patch 0097 — native-ELF CMake lane (#619, secondary artifact of #613)
+
+Makes the vendored tree buildable as a standard Linux ELF llama-server via
+plain cmake + g++/nvcc (rpath libggml-cuda.so; clean gdb/nsys), alongside the
+cosmocc Makefile lane which is UNTOUCHED (BUILD.mk lists sources explicitly
+and never reads CMakeLists; the new stub TU is CMake-lane-only).
+
+- 5 CMakeLists surgical hooks: add opencoti TUs absent from upstream source
+  lists (ggml-turbo-quant.c, ggml-neo-pipeline.cpp, ggml-iqk-flash-attn.cpp
+  -> ggml-base; dca.cpp -> llama; pcie-profile.cpp -> common; ELF stubs ->
+  ggml-cpu) + compile the Mozilla-overlay server.cpp with -Dmain=llama_server
+  (upstream defines llama_server directly; the cosmo lane renames at build).
+- NEW ggml/src/ggml-cpu/opencoti-elf-stubs.cpp: return-false stubs for the
+  cosmocc-bound llamafile/sgemm.cpp CPU fastpath dispatchers (mixmul,
+  mixmul_needs, mixmul_iqk, fa_* helpers) — every call site falls back to the
+  upstream generic path (same contract as fa_helpers_unsupported.cpp), so the
+  ELF build loses only CPU fastpaths, never correctness.
+
+Build (bs2-proven 2026-07-05): cmake -S llama.cpp -B build-elf -DGGML_CUDA=ON
+-DGGML_CUDA_FA_ALL_QUANTS=ON (REQUIRED — turbo/TCQ FA-VEC instances are only
+globbed under this flag; cosmo lane always sets it) -DCMAKE_CUDA_ARCHITECTURES=120
+-DCMAKE_BUILD_TYPE=Release -DLLAMA_CURL=OFF -DGGML_NATIVE=OFF.
+Gate: A4B -ngl 99 boot, dual-ctx assistant-MTP accept 0.764, 310.2 tps decode.
+
+--- a/llama.cpp/ggml/src/CMakeLists.txt
++++ b/llama.cpp/ggml/src/CMakeLists.txt
+@@ -205,6 +205,9 @@
+             ggml-threading.cpp
+             ggml-threading.h
+             ggml-quants.c
++            ggml-turbo-quant.c    # opencoti-hook: native-elf-cmake (#619) — turbo/TCQ CPU quant, in cosmo BUILD.mk but absent here
++            ggml-neo-pipeline.cpp # opencoti-hook: native-elf-cmake (#619) — NEO pipeline control (pure std)
++            ggml-iqk-flash-attn.cpp # opencoti-hook: native-elf-cmake (#619) — iqk FA enable flag (pure ggml)
+             ggml-quants.h
+             gguf.cpp)
+ 
+--- a/llama.cpp/src/CMakeLists.txt
++++ b/llama.cpp/src/CMakeLists.txt
+@@ -18,6 +18,7 @@
+             llama-context.cpp
+             llama-cparams.cpp
+             llama-grammar.cpp
++            dca.cpp               # opencoti-hook: native-elf-cmake (#619) — DCA (matches neither llama-*.cpp list nor models/ glob)
+             llama-graph.cpp
+             llama-hparams.cpp
+             llama-impl.cpp
+--- a/llama.cpp/common/CMakeLists.txt
++++ b/llama.cpp/common/CMakeLists.txt
+@@ -66,6 +66,7 @@
+     chat.cpp
+     chat.h
+     common.cpp
++    pcie-profile.cpp # opencoti-hook: native-elf-cmake (#619) — PCIe bandwidth profile (rolling-KV)
+     common.h
+     console.cpp
+     console.h
+--- a/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
++++ b/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
+@@ -52,6 +52,7 @@
+         ggml-cpu/vec.cpp
+         ggml-cpu/ops.h
+         ggml-cpu/ops.cpp
++        ggml-cpu/opencoti-elf-stubs.cpp # opencoti-hook: native-elf-cmake (#619) — sgemm.cpp dispatcher stubs (cosmo lane uses real sgemm.cpp)
+         )
+ 
+     target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)
+--- a/llama.cpp/tools/server/CMakeLists.txt
++++ b/llama.cpp/tools/server/CMakeLists.txt
+@@ -31,6 +31,11 @@
+ 
+ set(TARGET llama-server-impl)
+ 
++# opencoti-hook: native-elf-cmake (#619) — the Mozilla overlay's server.cpp defines
++# `int main` (the cosmocc Makefile lane renames it at compile time); upstream
++# defines `llama_server` directly. Rename here so main.cpp's wrapper links.
++set_source_files_properties(server.cpp PROPERTIES COMPILE_DEFINITIONS "main=llama_server")
++
+ add_library(${TARGET}
+     server.cpp
+     server-http.cpp
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cpu/opencoti-elf-stubs.cpp
+@@ -0,0 +1,57 @@
++// opencoti-hook: native-elf-cmake (#619) — see docs/features/llamafile_build.md
++//
++// CMake(native-ELF)-lane-ONLY stubs for the llamafile/sgemm.cpp CPU fastpath
++// dispatchers. sgemm.cpp includes <cosmo.h> (cosmocc runtime CPUID/dispatch)
++// and cannot compile under plain g++; every call site treats a false return
++// as "not handled, use the upstream ggml helper" (ggml-cpu.c:1723 mixmul_iqk,
++// ops.cpp:8519/8560 flash-attn helpers — same contract as Mozilla's own
++// fa_helpers_unsupported.cpp), so these stubs are behavior-safe: the native
++// ELF build loses only the iqk/tinyBLAS *CPU* fastpaths, never correctness.
++// The cosmocc Makefile lane never compiles this file (BUILD.mk lists sources
++// explicitly) and keeps the real sgemm.cpp implementations.
++
++#include <cstddef>
++#include <cstdint>
++
++struct ggml_compute_params;
++struct ggml_tensor;
++
++extern "C" {
++
++// Top-level tinyBLAS MoE mixmul (ggml-cpu.c:1592, GGML_USE_LLAMAFILE):
++// false -> generic mul_mat_id path. _needs sizes its work buffer
++// (ggml-cpu.c:3002): 0 is consistent with mixmul never running.
++bool llamafile_mixmul(const struct ggml_compute_params *,
++                      const struct ggml_tensor *,
++                      const struct ggml_tensor *,
++                      const struct ggml_tensor *,
++                      struct ggml_tensor *) {
++    return false;
++}
++
++size_t llamafile_mixmul_needs(const struct ggml_tensor *,
++                              const struct ggml_tensor *,
++                              const struct ggml_tensor *) {
++    return 0;
++}
++
++bool llamafile_mixmul_iqk(long, long, long, int, int,
++                          const void *, const void *, float *,
++                          long, long, const void *, int, int) {
++    return false;
++}
++
++bool llamafile_fa_vec_dot_f16(int, float *, const void *, const void *) {
++    return false;
++}
++
++bool llamafile_fa_fp16_to_fp32_row(const void *, float *, int64_t) {
++    return false;
++}
++
++bool llamafile_fa_simd_gemm(float *, const float *, const float *,
++                            int, int, int) {
++    return false;
++}
++
++} // extern "C"

patches/0098-rolling-kv-lse-decode.patch

@@ -0,0 +1,97 @@
+opencoti patch 0098 — rolling-KV: dst_lse emission for head_dim<=256 decode (bug-1843, task #623)
+
+Extends the Stage-3a LSE channel (opencoti_fattn_dst_lse, drained by
+launch_fattn's flash_attn_combine_results / stream-k fixup) to head_dim<=256
+n_q==1 decode at all three streaming-FA call sites (S3b overlap, S3a
+single-stream, S2 resident-view). Previously only head_dim>256 armed the
+channel; Qwen-class D128 decode fell through to the per-tile
+streaming_lse_kernel recompute — a <<<n_rows,WARP_SIZE>>> full key-rescan per
+tile that was MEASURED as ~80% of spill-decode wall (bug-1843 A/B on the 3090:
+1.14 tps base vs 6.05 tps OPENCOTI_LSE_NOOP, Qwen3-8B-Q8_0 ctx40960 vt14000).
+
+Mechanism: after the per-tile FA call, `lse_emitted = decode_lse &&
+opencoti_fattn_dst_lse_written` — the written flag reports whether a finalize
+(stream-k fixup or parallel-blocks combine) actually ran; the recompute now
+fires only when the dispatch resolved to a no-finalize single-block path. The
+combine emits the identical per-row value (kqmax + logf(denominator), row
+order h + n_head*(qi + n_q*b)) the recompute produced. The `this_kv >
+2*FATTN_KQ_STRIDE` guard is retained everywhere (bug-263 conservatism).
+Prefill and D>256 decode are behavior-identical.
+
+Gates (2026-07-05, 3090, DSO df46dc36):
+- perf: spill_base 1.14 -> 6.04 tps (5.3x, at the NOOP ceiling), needle
+  PRESENT; resident 42.5 tps unchanged; OPENCOTI_LSE_NOOP now speed-inert
+  (6.02) proving the recompute is structurally off the path.
+- correctness: teacher-forced logit-equiv (Qwen3-8B D128, ours-resident REF vs
+  POSITION_WINDOW 256/16384 cells resident): real_frac=0.0, divergences=0,
+  frac_full_agree=1.0, mean_tv 0.0068 -> PASS.
+
+DSO-only (fattn.cu; no header change, no host rebuild needed).
+
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -1596,11 +1596,20 @@
+                     // ntiles_KV=16 at 512, not 2). `>` is kept only because it is no worse
+                     // than `>=`; the real mechanism is unisolated. Real Stage-3c tail tiles
+                     // can be ≤512 at decode → MUST resolve before relying on this path.
+-                    const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE);
++                    // opencoti-hook: f5-rolling-kv bug-1843 — head_dim<=256 (Qwen-class VEC
++                    // decode) now ALSO arms the channel: the parallel-blocks combine emits
++                    // the per-row lse (Stage-3a formula in flash_attn_combine_results), and
++                    // opencoti_fattn_dst_lse_written reports whether a finalize actually ran
++                    // (T87.pD-opt O2 pattern) so the streaming_lse recompute below fires only
++                    // when the dispatch resolved to a no-finalize single-block path. The
++                    // recompute was measured at ~80% of spill-decode cost (1.14 vs 6.05 tps
++                    // A/B via OPENCOTI_LSE_NOOP).
++                    const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE);
+                     if (decode_lse) opencoti_fattn_dst_lse = lse_t;
+                     ggml_cuda_flash_attn_ext(ctx, &dst_t);
++                    const bool lse_emitted = decode_lse && opencoti_fattn_dst_lse_written;
+ 
+-                    if (!decode_lse) {
++                    if (!lse_emitted) {
+                         // opencoti F5 bug-1840 fix (b) DIAGNOSTIC — OPENCOTI_LSE_NOOP skips
+                         // the per-tile streaming_lse recompute (a low-parallelism
+                         // <<<n_rows,32>>> rescan run once per tile because Qwen/head_dim≤256
+@@ -1727,11 +1736,14 @@
+                 // (softcap-aware). The `> 2*FATTN_KQ_STRIDE` boundary is NOT a fix for
+                 // bug-263 (UNRESOLVED: decode tile_kv_full==512 corrupts on either lse
+                 // path) — kept only as no-worse-than-`>=`. See .wolf/buglog.json.
+-                const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE);
++                // bug-1843: head_dim<=256 also arms the channel; recompute only when no
++                // finalize ran (opencoti_fattn_dst_lse_written) — see the S3b note above.
++                const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE);
+                 if (decode_lse) opencoti_fattn_dst_lse = lse_t;
+                 ggml_cuda_flash_attn_ext(ctx, &dst_t);
++                const bool lse_emitted = decode_lse && opencoti_fattn_dst_lse_written;
+ 
+-                if (!decode_lse) {
++                if (!lse_emitted) {
+                     streaming_lse_kernel<<<n_rows, WARP_SIZE, 0, stream>>>(
+                         (const char *)Q->data, sK,
+                         mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr,
+@@ -1764,7 +1776,10 @@
+             // boundary is NOT a fix for bug-263 (UNRESOLVED: decode tile_kv_full==512
+             // corrupts on either lse path) — kept only as no-worse-than-`>=`. See the
+             // S3b overlap path's full note + .wolf/buglog.json.
+-            const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE);
++            // bug-1843: head_dim<=256 also arms the channel; recompute only when no
++            // finalize ran (opencoti_fattn_dst_lse_written) — see the S3b note above.
++            const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE);
++            bool lse_emitted = false;
+             if (this_kv > 0) {
+                 // FA over the tile's key-slice → O_t (stock kernel, untouched).
+                 ggml_tensor Kt = *K; Kt.ne[1] = this_kv; Kt.data = (char *)K->data + (size_t)kv0*K->nb[1];
+@@ -1784,10 +1799,11 @@
+                 dst_t.data   = O_tiles.ptr + (size_t)t * nel;
+                 if (decode_lse) opencoti_fattn_dst_lse = lse_t;
+                 ggml_cuda_flash_attn_ext(ctx, &dst_t);
++                lse_emitted = decode_lse && opencoti_fattn_dst_lse_written;
+             }
+ 
+             // lse_t (this_kv==0 → empty tile: kernel writes -inf, combine ignores it).
+-            if (!decode_lse) {
++            if (!lse_emitted) {
+                 streaming_lse_kernel<<<n_rows, WARP_SIZE, 0, stream>>>(
+                     (const char *)Q->data,
+                     (const char *)K->data + (size_t)kv0*K->nb[1],

patches/0099-dca-gemma4-wiring.patch

@@ -0,0 +1,140 @@
+opencoti patch 0099 — DCA: wire Gemma-4 builders (bug-2118, task #626 unblock)
+
+`--dca on` was a SILENT NO-OP on all Gemma-4 archs: 0078-dca.patch wired only
+the Qwen builders; dca.cpp's iswa build_attn_dca overload sat uncalled by
+gemma4.cpp / gemma4-assistant.cpp. Consequence: every historical Gemma x DCA
+eval measured DCA-off (contamination catalog: docs/evaluations bug-2118 notes).
+
+Wiring: build_attn_inp_dca on the iswa base cache; per-layer dca_l gates
+GLOBAL (non-SWA) layers only; Q/K rope moves inside build_attn_dca via the
+dca_rope bundle (chunked-regime rope must happen per-band); the assistant's
+Q-only shared-KV path passes (Qcur, nullptr, nullptr). SWA layers and DCA-off
+are byte-identical (else-branch keeps the original build_attn call).
+
+Gate (bs2, A4B + assistant, 32k, 24k prompt): INTRA regime (chunk==ctx) text
+byte-identical to DCA-off with identical accept (f16 0.677 both; q8q4 text
+identical); cross-chunk f16 accept 0.939, needle PRESENT.
+
+--- a/llama.cpp/src/models/gemma4.cpp
++++ b/llama.cpp/src/models/gemma4.cpp
+@@ -1,4 +1,6 @@
+ #include "models.h"
++#include "dca.h" // opencoti F5 dca bug-2118
++#include "llama-kv-cache-iswa.h" // bug-2118: get_base() on the iswa mctx
+ 
+ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) {
+     hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+@@ -161,6 +163,14 @@
+     // TODO: is causal == true correct? might need some changes
+     auto * inp_attn = build_attn_inp_kv_iswa();
+ 
++    // opencoti F5 dca bug-2118 — Gemma-4 DCA wiring. 0078 wired only the Qwen builders;
++    // the iswa build_attn_dca overload (dca.cpp) existed uncalled, so '--dca on' was a
++    // silent no-op on Gemma. DCA applies to the GLOBAL (non-SWA) layers only (the SWA
++    // window is <= chunk already); the overload stores PRE-rope K + handles has_kv==false
++    // shared-KV globals (null k/v -> read the earlier layer's cache).
++    const bool use_dca = cparams.dca_enabled;
++    llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx->get_base()) : nullptr;
++
+     ggml_tensor * inp_out_ids = build_inp_out_ids();
+ 
+     // opencoti-hook: gemma4-mtp-hidden (bug-858 dual-context MTP) — when a context extracts the
+@@ -202,6 +212,10 @@
+             freq_factors = model.layers[il].rope_freqs;
+         }
+ 
++        // opencoti F5 dca bug-2118 — global layers route through build_attn_dca (which
++        // ropes Q per regime and pre-ropes K at insert), so skip the standard pre-rope.
++        const bool dca_l = use_dca && !hparams.is_swa(il);
++
+         // Q projection (shared for both non-KV and KV layers)
+         // this is to mirror Gemma4Attention in pytorch code
+         ggml_tensor * Qcur;
+@@ -214,9 +228,11 @@
+             Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
+             cb(Qcur, "Qcur_normed", il);
+ 
++            if (!dca_l) { // bug-2118: DCA ropes Q per regime in build_attn_dca
+             Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                                  ext_factor, attn_factor, beta_fast, beta_slow);
+             cb(Qcur, "Qcur_pos", il);
++            }
+         }
+ 
+         // self-attention
+@@ -238,14 +254,28 @@
+             cb(Kcur, "Kcur_normed", il);
+             cb(Vcur, "Vcur_normed", il);
+ 
++            if (!dca_l) { // bug-2118: DCA pre-ropes K at insert in build_attn_dca
+             Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                                  ext_factor, attn_factor, beta_fast, beta_slow);
+ 
+             cb(Kcur, "Kcur_pos", il);
++            }
+ 
++            if (dca_l) { // bug-2118 (wo_s dropped like the qwen3 DCA path — null on these GGUFs)
++                const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors };
++                cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr,
++                        Qcur, Kcur, Vcur, rp, hparams.f_attention_scale, il);
++            } else {
+             cur = build_attn(inp_attn, model.layers[il].wo,
+                     nullptr, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
+                     hparams.f_attention_scale, il);
++            }
++        } else if (dca_l) {
++            // bug-2118: shared-KV global layer under DCA — Q-only read of the earlier
++            // layer's DCA-formatted cache (null k/v -> no store).
++            const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors };
++            cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr,
++                    Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, il);
+         } else {
+             // reuse KV cache of earlier layers
+             cur = build_attn(inp_attn,
+--- a/llama.cpp/src/models/gemma4-assistant.cpp
++++ b/llama.cpp/src/models/gemma4-assistant.cpp
+@@ -1,5 +1,7 @@
+ // opencoti F5 M6-S4 mtp
+ #include "models.h"
++#include "dca.h" // opencoti F5 dca bug-2118
++#include "llama-kv-cache-iswa.h" // bug-2118: get_base() on the iswa mctx
+ 
+ #include <cmath>
+ #include <vector>
+@@ -190,6 +192,13 @@
+         ggml_tensor * inp_pos     = build_inp_pos();
+         ggml_tensor * inp_out_ids = build_inp_out_ids();
+ 
++        // opencoti F5 dca bug-2118 — when the TARGET runs DCA, the drafter's global layers
++        // read the target's DCA-formatted (pre-roped, position-remapped) shared K/V, so its
++        // Q must be roped per regime through the same build_attn_dca path. Without this the
++        // drafter reads remapped keys with raw-extrapolated Q beyond native ctx -> accept 0.
++        const bool use_dca = cparams.dca_enabled;
++        llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx->get_base()) : nullptr;
++
+         ggml_tensor * inpL = cur;
+ 
+         for (int il = 0; il < n_layer; ++il) {
+@@ -212,13 +221,22 @@
+             cb(Qcur, "Qcur_normed", il);
+ 
+             ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs;
++            const bool dca_l = use_dca && !is_swa; // bug-2118
++            if (!dca_l) {
+             Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig,
+                                  freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+             cb(Qcur, "Qcur_pos", il);
++            }
+ 
+             // KV-less cross-attention into the shared/aliased target K/V (null k_cur/v_cur -> no store).
++            if (dca_l) { // bug-2118: Q-only per-regime read of the target's DCA cache
++                const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors };
++                cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr,
++                        Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, il);
++            } else {
+             cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr,
+                     Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
++            }
+ 
+             if (il == n_layer - 1 && inp_out_ids) {
+                 cur  = ggml_get_rows(ctx0, cur,  inp_out_ids);

patches/0100-dca-single-chunk-defer.patch

@@ -0,0 +1,36 @@
+opencoti patch 0100 — DCA: defer quant lift in the single-chunk regime (bug-2119)
+
+dca.cpp gated defer_quant_lift on !single_chunk, so the single-chunk (INTRA)
+regime whole-cache dca_lift_to_f16'd the quantized K/V EVERY forward — a
+per-decode-step full-cache dequant that collapsed A4B 32k q8K/q4V decode to
+4.69 tps. The comment assumed n_kv<=chunk meant "tiny cache"; at chunk 32768
+it is the entire context. Stock ggml_flash_attn_ext converts scalar-quant
+natively, so the single-chunk branch defers exactly like analytical-bands.
+
+Gate (bs2, A4B q8K/q4V 32k chunk-32768): decode 4.69 -> 73.74 tps
+(DCA-off 77.13); INTRA text remains byte-identical to DCA-off.
+
+--- a/llama.cpp/src/dca.cpp
++++ b/llama.cpp/src/dca.cpp
+@@ -255,16 +255,16 @@
+     // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see
+     // ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case), so pass the quantized cache THROUGH instead of
+     // whole-cache materializing it to f16 via dca_lift_to_f16 every forward (the prefill cliff). Gate:
+-    //   - !single_chunk: only the fused op carries the in-launch convert; single_chunk uses stock
+-    //     ggml_flash_attn_ext (launch_fattn already converts scalar-quant natively) and runs only while
+-    //     n_kv <= chunk (a tiny cache) so its lift cost is negligible — keep it on the safe lift path.
+-    //   - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read.
++    //   - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read. bug-2119: the
++    //     single_chunk branch defers too — it uses stock ggml_flash_attn_ext, whose launch_fattn
++    //     converts scalar-quant natively (the ordinary DCA-off decode path). The old !single_chunk gate
++    //     assumed n_kv <= chunk means "tiny cache, lift is negligible", but chunk can be 32k+: the
++    //     whole-cache two-leg graph lift ran EVERY decode step (A4B 32k/chunk-32768 q8q4: 76 -> 4.7 tps).
+     //   - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the
+     //     permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift.
+     // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output.
+     const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2];
+     const bool defer_quant_lift =
+-        !inp_dca->single_chunk &&
+         dca_is_inkernel_liftable(k_raw->type) &&
+         dca_is_inkernel_liftable(v_raw->type) &&
+         !v_trans_raw;

patches/0101-kv-attn-rot-inherit.patch

@@ -0,0 +1,49 @@
+opencoti patch 0101 — shared draft cache inherits attention-rotation state (bug-2120)
+
+llamafile 0.10.3 attention rotation (Hadamard-rotate K/V rows before quantized
+store when the cache type is quantized && head_dim%64==0; readers rotate Q and
+un-rotate the output via attn_inp_k_rot/v_rot) is computed per-cache from its
+OWN type_k/type_v and accumulated head dims. The dual-context assistant draft
+cache is created with f16 types AND its shared layers skip the head-dim
+accumulation, so rotation silently disabled: the drafter read the target's
+ROTATED quant K/V with an unrotated Q -> decorrelated attention, flat finite
+logits, accept ~0.000 with ANY quant target KV (f16/f16 unaffected; Qwen NextN
+unaffected — it reads through the target's own cache object).
+
+Fix: in the ctor share branch, inherit attn_rot_k/v, n_embd_head_*_all and the
+Hadamard tables from the source cache, overriding the local computation. The
+generic iswa build_attn then applies the rotation on the drafter's Q-only path.
+
+Gate (bs2, A4B + assistant Q8_0 drafter, 32k, 24k prompt): accept
+k8_v16 0.000 -> 0.641; k8_v4 (production PolyKV) 0.000 -> 0.729; f16 control
+0.677 unchanged.
+
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -1106,6 +1106,26 @@
+         }
+     }
+ 
++    // opencoti bug-2120: a shared (dual-context drafter) cache must READ with the same
++    // attention-rotation state the SOURCE cache used to WRITE. The local computation above
++    // sees this cache's own type_k/type_v (f16 for the drafter) and n_embd_head_*_all == 0
++    // (shared layers `continue` before accumulating), so it always disables rotation — the
++    // drafter then reads Hadamard-rotated quant K/V with an unrotated Q and un-unrotated
++    // output: decorrelated attention, accept ~0 with ANY quant target KV (f16/f16 unaffected
++    // because no rotation is applied at all). Inherit flags + head-dim extents + Hadamard
++    // tables from `other` so build_input_k_rot/v_rot emit the matching rotation inputs.
++    if (share && other) {
++        attn_rot_k        = other->attn_rot_k;
++        attn_rot_v        = other->attn_rot_v;
++        n_embd_head_k_all = other->n_embd_head_k_all;
++        n_embd_head_v_all = other->n_embd_head_v_all;
++        attn_rot_hadamard = other->attn_rot_hadamard;
++        if (attn_rot_k || attn_rot_v) {
++            LLAMA_LOG_INFO("%s: shared cache inherits attn_rot_k=%d attn_rot_v=%d from source (bug-2120)\n",
++                    __func__, (int) attn_rot_k, (int) attn_rot_v);
++        }
++    }
++
+     const char * LLAMA_KV_CACHE_DEBUG = getenv("LLAMA_KV_CACHE_DEBUG");
+     debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0;
+ }

patches/0102-fattn-bulk-h2d-scratch-ring.patch

@@ -0,0 +1,154 @@
+opencoti patch 0102 — streaming-FA bulk-H2D tail lift + persistent staging scratch ring (bug-2116, bug-2121)
+
+Two changes to the rolling-KV streaming-FA lift in fattn.cu:
+
+1. bug-2116 (perf): for a HOST quant tail source at decode (n_b==1), bulk-copy
+   the RAW QUANT bytes H2D once at full DMA rate into device scratch, then run
+   the nc-converter from device memory. The converter reading the pinned-host
+   tail via UVA at word granularity measured 10-16 GB/s vs 57-65 GB/s bulk on
+   bs2 — the UVA cliff behind the S0 spill-decode curve. Measured after-curve
+   (14B-1M q8q4 window 131k, decode tps): 800MiB 9.05->12.23, 1600 3.96->10.28,
+   6400 ~1->5.53.
+
+2. bug-2121 (correctness): both bulk paths (the new quant one and the #586 f16
+   one) staged through ggml_cuda_pool_alloc scratch FREED on scope exit while
+   the async memcpy+convert on the copy stream were still pending. Decode
+   (bug-1841 single stream) is stream-ordered and safe; PREFILL runs two
+   streams and the FA compute allocates from the SAME pool concurrently (MMA
+   stream-k fixup buffers, every tile) -> freed-but-pending scratch re-handed
+   cross-stream -> corrupted K/V tiles. Manifested with >=2 host-tail tiles
+   (>32768 exercised tail cells): garbage decode / needle-LOST + racy host
+   GPFs; <=1 tail tile clean. Fix: persistent per-slot scratch ring
+   (g_scrK/g_scrV[4], grow-only, worst-case tile_kv_full capacity so no
+   cudaMalloc inside captured decode graphs) — no pool sharing across streams.
+
+Gate (bs2, 14B-1M q8q4 window 131k): tails 1600/3200/6400 all needle=PRESENT,
+coherent text, zero crashes across three full 106k prefills (3200 previously
+crashed 2x + garbage decode; 6400 previously needle-LOST).
+
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -1350,6 +1350,30 @@
+         char * const slotK_base = g_slotK;
+         char * const slotV_base = g_slotV;
+ 
++        // opencoti-hook: f5-rolling-kv bug-2121 — PERSISTENT per-slot H2D staging scratch.
++        // The bulk-H2D lifts below (bug-2116 quant path + #586 f16 path) used
++        // ggml_cuda_pool_alloc scratch FREED on lambda/loop exit while the
++        // cudaMemcpyAsync + converter on the copy stream were still pending. On decode
++        // (single stream, bug-1841) reuse is stream-ordered and safe; on PREFILL the FA
++        // compute on ctx.stream() allocates from the SAME pool concurrently (MMA
++        // stream-k fixup buffers, every tile), so a freed-but-pending scratch gets
++        // re-handed to the other stream → cross-stream aliasing → corrupted K/V tiles.
++        // Manifested only with ≥2 host-tail tiles interleaving with compute (>32768
++        // exercised tail cells): 14B-1M q8q4 window-mode tail=3200/6400 MiB → garbage
++        // decode / needle-LOST (+ racy host GPFs); tail≤1600 (1 tail tile) clean. A
++        // persistent ring keyed by slot removes pool sharing entirely; within-ring WAR
++        // is stream-ordered (memcpy → convert → next lift's memcpy, all on cs).
++        static char * g_scrK[4] = {}; static size_t g_scrK_cap[4] = {};
++        static char * g_scrV[4] = {}; static size_t g_scrV_cap[4] = {};
++        auto scr_get = [](char ** ring, size_t * cap, int slot, size_t need) -> char * {
++            if (cap[slot] < need) {
++                if (ring[slot]) CUDA_CHECK(cudaFree(ring[slot]));   // cudaFree syncs pending users
++                CUDA_CHECK(cudaMalloc((void **)&ring[slot], need));
++                cap[slot] = need;
++            }
++            return ring[slot];
++        };
++
+         // R2-b S3b (#312): with >=2 slots the op runs a double-buffer ping-pong — the
+         // per-tile lift is issued on a dedicated copy_stream while FA/lse for the
+         // current tile run on ctx.stream(), so tile t+1's DMA overlaps tile t's
+@@ -1445,10 +1469,34 @@
+                     // below are unchanged. Unifies device + pinned-host (S3c) sources
+                     // (the kernel reads host via UVA). Slot is f16 → lse half-cast OK.
+                     // 3c-5: tK/tV + loc select the window (device) or tail (host) source.
+-                    to_fp16_k((const char *)tK->data + (size_t)loc*tK->nb[1], (half *)sK,
++                    // opencoti-hook: f5-rolling-kv bug-2116 — for a HOST quant source,
++                    // bulk-H2D the RAW QUANT bytes (cell-major block, 2.5× fewer bytes
++                    // than f16) into device scratch at full DMA rate, then convert from
++                    // device memory. Calling the converter on the host pointer made it
++                    // read the tail via UVA at word granularity (~10–16 GB/s measured
++                    // vs 57–65 GB/s bulk on bs2). n_b>1 keeps the UVA path (nb[3]
++                    // batch blocks are not spanned by the cell-major copy; unused in
++                    // the spill regime where ne[3]==1).
++                    const bool   bulkq  = tr.host && n_b == 1;
++                    const char * qsrcK = (const char *)tK->data + (size_t)loc*tK->nb[1];
++                    const char * qsrcV = (const char *)tV->data + (size_t)loc*tV->nb[1];
++                    if (bulkq) {
++                        // bug-2121: persistent per-slot scratch — NOT pool memory (see scr_get above).
++                        // Capacity = worst-case tile (tile_kv_full), not this_kv: this_kv grows per
++                        // decode token, and a realloc inside a captured decode graph would fail.
++                        const size_t kqblk = (size_t)this_kv * tK->nb[1];
++                        const size_t vqblk = (size_t)this_kv * tV->nb[1];
++                        char * qscrK = scr_get(g_scrK, g_scrK_cap, slot, (size_t)tile_kv_full * tK->nb[1]);
++                        char * qscrV = scr_get(g_scrV, g_scrV_cap, slot, (size_t)tile_kv_full * tV->nb[1]);
++                        CUDA_CHECK(cudaMemcpyAsync(qscrK, qsrcK, kqblk, cudaMemcpyDefault, cs));
++                        CUDA_CHECK(cudaMemcpyAsync(qscrV, qsrcV, vqblk, cudaMemcpyDefault, cs));
++                        qsrcK = qscrK;
++                        qsrcV = qscrV;
++                    }
++                    to_fp16_k(qsrcK, (half *)sK,
+                               head_dim, this_kv, n_head_kv, n_b,
+                               (int64_t)(tK->nb[1]/ts_k), (int64_t)(tK->nb[2]/ts_k), (int64_t)(tK->nb[3]/ts_k), cs);
+-                    to_fp16_v((const char *)tV->data + (size_t)loc*tV->nb[1], (half *)sV,
++                    to_fp16_v(qsrcV, (half *)sV,
+                               dv, this_kv, n_head_kv, n_b,
+                               (int64_t)(tV->nb[1]/ts_v), (int64_t)(tV->nb[2]/ts_v), (int64_t)(tV->nb[3]/ts_v), cs);
+                 } else
+@@ -1468,17 +1516,20 @@
+                     const bool bulk = use_2d && tr.host;
+                     const size_t kblk = (size_t)this_kv * tK->nb[1];   // contiguous cell-major span
+                     const size_t vblk = (size_t)this_kv * tV->nb[1];
+-                    ggml_cuda_pool_alloc<char> scrK(pool, bulk ? kblk : 1);
+-                    ggml_cuda_pool_alloc<char> scrV(pool, bulk ? vblk : 1);
+                     const char * baseK;
+                     const char * baseV;
+                     if (bulk) {
++                        // bug-2121: persistent per-slot scratch — NOT pool memory (see scr_get above).
++                        // Sequential bb iterations reuse it WAR-safely (copy/repack both on cs).
++                        // Worst-case tile capacity: see the quant path above (capture-safe growth).
++                        char * scrK = scr_get(g_scrK, g_scrK_cap, slot, (size_t)tile_kv_full * tK->nb[1]);
++                        char * scrV = scr_get(g_scrV, g_scrV_cap, slot, (size_t)tile_kv_full * tV->nb[1]);
+                         const char * hK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1];
+                         const char * hV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1];
+-                        CUDA_CHECK(cudaMemcpyAsync(scrK.ptr, hK, kblk, cudaMemcpyDefault, cs));   // one full-BW H2D
+-                        CUDA_CHECK(cudaMemcpyAsync(scrV.ptr, hV, vblk, cudaMemcpyDefault, cs));
+-                        baseK = scrK.ptr;   // scratch is cell-major, cell `loc` at offset 0
+-                        baseV = scrV.ptr;
++                        CUDA_CHECK(cudaMemcpyAsync(scrK, hK, kblk, cudaMemcpyDefault, cs));   // one full-BW H2D
++                        CUDA_CHECK(cudaMemcpyAsync(scrV, hV, vblk, cudaMemcpyDefault, cs));
++                        baseK = scrK;   // scratch is cell-major, cell `loc` at offset 0
++                        baseV = scrV;
+                     } else {
+                         baseK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1];
+                         baseV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1];
+@@ -1677,10 +1728,26 @@
+                 if (!all_f16) {
+                     // S3d: one nc-converter launch per tile → packed f16 slot (see
+                     // the overlap path for the layout proof). Source strides s=nb/ts.
+-                    to_fp16_k((const char *)K->data + (size_t)kv0*K->nb[1], (half *)sK,
++                    // bug-2116: host quant source → bulk-H2D the raw quant block at
++                    // DMA rate + convert from device scratch; direct converter-on-host
++                    // reads via UVA at word granularity (see the overlap-path note).
++                    const bool   bulkq  = host_src && n_b == 1;
++                    const size_t kqblk  = bulkq ? (size_t)this_kv * K->nb[1] : 1;
++                    const size_t vqblk  = bulkq ? (size_t)this_kv * V->nb[1] : 1;
++                    ggml_cuda_pool_alloc<char> qscrK(pool, kqblk);
++                    ggml_cuda_pool_alloc<char> qscrV(pool, vqblk);
++                    const char * qsrcK = (const char *)K->data + (size_t)kv0*K->nb[1];
++                    const char * qsrcV = (const char *)V->data + (size_t)kv0*V->nb[1];
++                    if (bulkq) {
++                        CUDA_CHECK(cudaMemcpyAsync(qscrK.ptr, qsrcK, kqblk, cudaMemcpyDefault, stream));
++                        CUDA_CHECK(cudaMemcpyAsync(qscrV.ptr, qsrcV, vqblk, cudaMemcpyDefault, stream));
++                        qsrcK = qscrK.ptr;
++                        qsrcV = qscrV.ptr;
++                    }
++                    to_fp16_k(qsrcK, (half *)sK,
+                               head_dim, this_kv, n_head_kv, n_b,
+                               (int64_t)(K->nb[1]/ts_k), (int64_t)(K->nb[2]/ts_k), (int64_t)(K->nb[3]/ts_k), stream);
+-                    to_fp16_v((const char *)V->data + (size_t)kv0*V->nb[1], (half *)sV,
++                    to_fp16_v(qsrcV, (half *)sV,
+                               dv, this_kv, n_head_kv, n_b,
+                               (int64_t)(V->nb[1]/ts_v), (int64_t)(V->nb[2]/ts_v), (int64_t)(V->nb[3]/ts_v), stream);
+                 } else

patches/0103-kv-cpu-fa-probe-guard.patch

@@ -0,0 +1,67 @@
+opencoti patch 0103 — rolling-KV CPU-FA cost-probe guard (bug-2125)
+
+The Stage-3d-S2 rolling-KV tactic selector runs a synthetic CPU flash-attn
+"companion probe" (opencoti_fa_compute_probe_ms with on_cpu=true) purely to
+MODEL a CPU_FA_TAIL cost for its audit log. CPU_FA_TAIL is gated out of
+auto-select (#354) — the tail always streams on the GPU (POSITION_WINDOW) — so
+the probe's result is never used for the decision.
+
+That probe builds a real ggml_flash_attn_ext over the actual cache K/V types and
+runs it on the host CPU backend. For opencoti's CUDA-FA-VEC-only KV types (q6_0,
+turbo{2,3,4,8}_0, turbo{2,3}_tcq) the CPU vec_dot trait is NULL, so the CPU
+flash-attn dereferences a NULL fn-ptr → SIGSEGV at context-init (before any
+client request). Stock types (f16/q8_0/q4_0/q5_0/q5_1/bf16) carry a CPU vec_dot
+and survive, which is why only the custom-K tiers crashed under DCA at 256k/512k.
+
+Fix: gate the CPU probe on a host-CPU-FA-capability predicate
+(opencoti_cpu_fa_capable_ktype) so it is never built for a K type the CPU
+cannot process. Returning 0.0f just leaves CPU_FA_TAIL unmodeled; the selector
+keeps its GPU-stream default. The rolling window stays a pure-GPU path for every
+KV tier — no silent CPU fallback. Host-only change (no CUDA DSO impact).
+--- a/llama.cpp/src/llama-kv-cache.cpp
++++ b/llama.cpp/src/llama-kv-cache.cpp
+@@ -2795,6 +2795,28 @@
+ // build_attn_mha FA contract (post-permute q[hd,n_q,n_head], k/v[hd,n_kv,n_head_kv],
+ // F16 mask). Never aborts boot: every shortfall (no GPU, unsupported op, non-GQA
+ // geometry, alloc miss) returns 0.0f and the selector keeps the stream default.
++//
++// opencoti bug-2125 SAFEGUARD: can the HOST CPU flash-attn path
++// (ggml_compute_forward_flash_attn_ext_f16) process a cache of this K type? It reads K via the
++// K-type's CPU vec_dot trait, which is NULL for opencoti's CUDA-FA-VEC-only KV types (q6_0,
++// turbo{2,3,4,8}_0, turbo{2,3}_tcq — GPU in-register readers, no CPU vec_dot / dp kernel). Running
++// CPU flash-attn over such a K deref's a NULL fn-ptr → SIGSEGV. The rolling-KV tail is ALWAYS
++// streamed on the GPU (POSITION_WINDOW; CPU_FA_TAIL is gated out of auto-select, #354), so the ONLY
++// place a host CPU flash-attn is ever built is the Stage-3d-S2 cost-model probe below — and its
++// result is audit-log only. Gate the CPU probe on this predicate so the rolling window stays a
++// pure-GPU path with NO silent CPU fallback for ANY KV tier. Stock ggml types (which DO carry a CPU
++// vec_dot) are the allow-list; every opencoti custom KV type is excluded.
++static bool opencoti_cpu_fa_capable_ktype(enum ggml_type t) {
++    switch (t) {
++        case GGML_TYPE_F32:  case GGML_TYPE_F16:  case GGML_TYPE_BF16:
++        case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1:
++        case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1:
++        case GGML_TYPE_Q8_0:
++            return true;  // stock ggml types: have a host CPU vec_dot
++        default:
++            return false; // opencoti custom KV (q6_0 / turbo* / turbo*_tcq): CUDA-only, NULL CPU vec_dot
++    }
++}
+ static float opencoti_fa_compute_probe_ms(
+         const llama_model & model, const llama_hparams & hparams,
+         int32_t il, ggml_type type_k, ggml_type type_v, int64_t bench_cells,
+@@ -2823,6 +2845,15 @@
+         return 0.0f; // device-type mismatch; selector falls back (stream default / no CPU model)
+     }
+ 
++    // opencoti bug-2125 SAFEGUARD: never build a host CPU flash-attn over a K type the CPU cannot
++    // process (NULL vec_dot → SIGSEGV). This is audit-only (the tail always streams on GPU), so
++    // returning 0.0f just leaves CPU_FA_TAIL unmodeled — the selector keeps the GPU-stream default.
++    // Keeps the rolling window a pure-GPU path for q6_0 / turbo* / turbo*_tcq (and any future
++    // CUDA-only KV tier), with no silent CPU fallback.
++    if (on_cpu && !opencoti_cpu_fa_capable_ktype(type_k)) {
++        return 0.0f;
++    }
++
+     const int64_t hd_k      = (int64_t) hparams.n_embd_head_k(il);
+     const int64_t hd_v      = (int64_t) hparams.n_embd_head_v(il);
+     const int64_t n_head    = (int64_t) hparams.n_head(il);

patches/0104-fattn-q6q4-vec-instance.patch

@@ -0,0 +1,71 @@
+opencoti patch 0104 — q6_0-K / q4_0-V FA-VEC instance (bug-2125 / #627)
+
+q6_0-K + q4_0-V (strong-K, cheap-V — the design invariant K precision >= V) is a
+first-class asymmetric KV combo. Under GGML_CUDA_FA_ALL_QUANTS the kernel
+selector admits K!=V and picks the VEC path, but no (q6_0, q4_0) FA-VEC instance
+existed, so the D<=256 dispatch (Gemma-4 local/SWA layers — the global layers
+use the DCA fused kernel's in-launch to_fp16_nc) fell through to
+GGML_ABORT("fatal error") at ggml_cuda_flash_attn_ext_vec, crashing at boot.
+
+Fix (mirrors the q6_0-q5_0 pattern, #512/#81): add the instance TU
+fattn-vec-instance-q6_0-q4_0.cu (head_dim 64/128/256), the
+FATTN_VEC_CASES_ALL_D(Q6_0, Q4_0) dispatch case, and the build-functions.sh 3c
+default-build inclusion (so the symbol is present without FA_ALL_QUANTS too).
+Verified bs2 256k DCA-on: boots HEALTHY (was abort) + niah_single_1 = 100.0.
+CUDA DSO change (needs a cuda rebuild). Only the q6_0/q4_0 direction is added —
+reverse q4_0/q6_0 is intentionally NOT added (K must stay the higher-bit type).
+--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+@@ -353,13 +353,17 @@
+     // ik_llama.cpp ships for q6_0 KV), registered OUTSIDE the FA_ALL_QUANTS block so they compile in
+     // the DEFAULT build — the matching instance .cu files are added unconditionally in
+     // collect_gpu_sources (build-functions.sh section 3c), like the turbo K==V instances below.
+-    // q6_0-q6_0 is the symmetric (default-reachable) cell; the 4 mixed cells are reachable only under
++    // q6_0-q6_0 is the symmetric (default-reachable) cell; the 5 mixed cells are reachable only under
+     // FA_ALL_QUANTS (the K!=V early-out in ggml_cuda_get_best_fattn_kernel), same as q5_0's mixed pairs.
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q6_0)
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q6_0)
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q5_0)
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_F16)
+     FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16,  GGML_TYPE_Q6_0)
++    // opencoti-hook: q6_0 KV (#627/bug-2125) — strong-K q6_0 + cheap-V q4_0 asymmetric cell. Under
++    // FA_ALL_QUANTS the selector admits q6_0-K/q4_0-V, so the stock FA-VEC dispatch (local SWA layers)
++    // needs this instance; without it the D<=256 dispatch fell through to the GGML_ABORT below.
++    FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q4_0)
+ 
+     // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec path
+     // (always registered, independent of GGML_CUDA_FA_ALL_QUANTS). turbo2/3/4 at head_dim ∈
+--- /dev/null
++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q4_0.cu
+@@ -0,0 +1,11 @@
++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — strong-K + cheap-V asymmetric cell (#627/bug-2125).
++// q6_0-K / q4_0-V is a first-class KV combo (6-bit K precision, 4-bit V footprint); the local SWA layers
++// read it through the stock FA-VEC path (the global layers use the DCA fused kernel's to_fp16_nc dequant),
++// so this instance must exist or ggml_cuda_flash_attn_ext_vec_case<D,q6_0,q4_0> is missing and the D<=256
++// dispatch aborts at fattn.cu (GGML_ABORT "fatal error"). Mirrors fattn-vec-instance-q6_0-q5_0.cu.
++
++#include "../fattn-vec.cuh"
++
++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0);
++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0);
++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0);
+--- a/llamafile/build-functions.sh
++++ b/llamafile/build-functions.sh
+@@ -162,7 +162,8 @@
+     fi
+ 
+     # 3c. opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). q6_0 is a scalar KV type read by the
+-    #     FA-VEC path; its 5 Anbeeld instances must compile in the DEFAULT build (same #468/bug-339
++    #     FA-VEC path; its 6 instances (5 Anbeeld + q6_0/q4_0 strong-K/cheap-V, #627) must compile in
++    #     the DEFAULT build (same #468/bug-339
+     #     reasoning as the turbo block above — the default build does NOT glob fattn-vec, so without
+     #     this the DSO lacks ggml_cuda_flash_attn_ext_vec_case<D,q6_0,*> and dlopen fails the instant
+     #     q6_0 KV reaches flash-attention). No-op under FA_ALL_QUANTS=1 (already globbed) and on a
+@@ -171,6 +172,7 @@
+         for f in "$ti_dir"/fattn-vec-instance-q6_0-q6_0.cu \
+                  "$ti_dir"/fattn-vec-instance-q8_0-q6_0.cu \
+                  "$ti_dir"/fattn-vec-instance-q6_0-q5_0.cu \
++                 "$ti_dir"/fattn-vec-instance-q6_0-q4_0.cu \
+                  "$ti_dir"/fattn-vec-instance-q6_0-f16.cu \
+                  "$ti_dir"/fattn-vec-instance-f16-q6_0.cu; do
+             [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f"