diff --git a/patches/0001-lazy-slot-context-defer.patch b/patches/0001-lazy-slot-context-defer.patch new file mode 100644 index 0000000000000000000000000000000000000000..09c048cdbcb1cd79c5fac57f08d32e2826c13ebb --- /dev/null +++ b/patches/0001-lazy-slot-context-defer.patch @@ -0,0 +1,247 @@ +From: opencoti +Subject: [PATCH 1/N] lazy slot-context allocation — Phase 1 (deferred zero-fill) + +Phase 1 of the F4 M3 lazy slot-context rollout. The eager +`ggml_backend_buffer_clear(buf, 0)` memset that commits every KV +buffer page to RSS at construction time is split out into a new +`ensure_cleared()` method invoked from inference entry points +(init_batch, prepare, update, state_read). + +The buffer's malloc happens eagerly at construction time so the +backend scheduler's `sched_reserve` (which runs at llama_context +construction, before any batch) can resolve `tensor->buffer` for +every KV tensor. On Linux, malloc is essentially free for unused +ranges — it reserves address space without committing pages. +Pages only commit to RSS when first written. The eager memset(0) +was what wrote every page, so removing it from the constructor +moves the RSS commit to the first real inference call. + +Effect for the production embedder (qwen3-0.6b, +`--parallel 2 --ctx-size 16384`, CPU mode): the KV cache's +~1,792 MiB no longer commits at process startup. Idle RSS drops +from ~3,478 MiB to roughly model + compute scratch only. First +request pays the memset cost (~1 s for 1.8 GiB), then warm-state +RSS matches the un-patched baseline. Phase 1 alone is a +cold-start win; Phases 2 (grow-on-demand) and 3 (shrink-on-idle) +target warm-state RSS. + +Smoke validation (Qwen2-Math-1.5B Q4_K_M, --parallel 2 -c 4096 +CPU mode): startup emits + + llama_kv_cache: CPU KV buffer size = 112.00 MiB + llama_kv_cache: KV cache zero-fill deferred until first batch + +First /v1/chat/completions request emits + + ensure_cleared: CPU KV buffer zero-fill = 112.00 MiB (deferred) + +and produces the expected response. + +Coverage of ensure_cleared() call sites: +- init_batch (inference entry) +- prepare (iswa wrapper calls prepare() directly on each inner + kv_cache without going through init_batch; see + llama-kv-cache-iswa.cpp:126-159) +- update (cross-stream seq_cp resolution + RoPE shift) +- state_read (saved-state restore writes tensor data) + +state_write is left const + un-cleared-safe: empty v_cells write +zero-cell-count records that state_read's +`if (cell_count == 0) continue` already handles. + +clear(data=true) on an un-cleared cache skips the buffer memset +deliberately — v_cells.reset() above already represents "no +data," and committing RSS just to zero it would defeat Phase 1's +purpose. + +The no_alloc model path is unchanged: zero-byte dummy buffers +are assigned per tensor at construction and trivially "cleared" +(zero-byte memset). `cleared` is set to true in that case. + +Bench: docs/decisions/0001-lazy-slot-context.md §6.2 Phase 1 row. +Milestone: F4 M3 — see docs/features/memory_embedder.md +Design: docs/decisions/0001-lazy-slot-context.md +Upstreaming: deferred; propose to llamafile + llama.cpp + maintainers after F4 M3's full phase-set ships and bench + numbers are public. This is a generally-useful patch. + +--- +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -250,7 +250,15 @@ llama_kv_cache::llama_kv_cache( + } + } + +- // allocate tensors and initialize the buffers to avoid NaNs in the padding ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // Eager malloc, deferred zero-fill. The address-space reservation done ++ // by ggml_backend_alloc_ctx_tensors_from_buft is essentially free on ++ // Linux — pages are not committed to RSS until they are first written. ++ // The `ggml_backend_buffer_clear(buf, 0)` memset below is what commits ++ // every page. By skipping it for the non-no_alloc path we keep the ++ // scheduler-visible tensor->buffer wiring intact (sched_reserve runs ++ // at construction time and asserts buffer_id >= 0) while moving the ++ // RSS commit to the first ensure_cleared() call. + for (auto & [buft, ctx] : ctx_map) { + ggml_backend_buffer_t buf; + if (model.hparams.no_alloc) { +@@ -259,7 +267,7 @@ llama_kv_cache::llama_kv_cache( + t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it + } + } else { +- buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer ++ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet) + } + if (!buf) { + throw std::runtime_error("failed to allocate buffer for kv cache"); +@@ -267,10 +275,23 @@ llama_kv_cache::llama_kv_cache( + + LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); + +- ggml_backend_buffer_clear(buf, 0); ++ if (model.hparams.no_alloc) { ++ // Dummy buffer is size 0; clearing is a no-op but keeps the ++ // upstream invariant that constructed buffers are zeroed. ++ ggml_backend_buffer_clear(buf, 0); ++ } ++ // else: leave the buffer un-cleared. ensure_cleared() will zero it ++ // on the first inference entry point that touches tensor data. ++ + ctxs_bufs.emplace_back(std::move(ctx), buf); + } + ++ if (model.hparams.no_alloc) { ++ cleared = true; ++ } else { ++ LLAMA_LOG_INFO("%s: KV cache zero-fill deferred until first batch\n", __func__); ++ } ++ + { + const size_t memory_size_k = size_k_bytes(); + const size_t memory_size_v = size_v_bytes(); +@@ -327,6 +348,22 @@ llama_kv_cache::llama_kv_cache( + debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0; + } + ++// opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++void llama_kv_cache::ensure_cleared() { ++ if (cleared) { ++ return; ++ } ++ ++ for (auto & [_, buf] : ctxs_bufs) { ++ LLAMA_LOG_INFO("%s: %10s KV buffer zero-fill = %8.2f MiB (deferred)\n", __func__, ++ ggml_backend_buffer_name(buf.get()), ++ ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0); ++ ggml_backend_buffer_clear(buf.get(), 0); ++ } ++ ++ cleared = true; ++} ++ + void llama_kv_cache::clear(bool data) { + for (uint32_t s = 0; s < n_stream; ++s) { + v_cells[s].reset(); +@@ -334,8 +371,14 @@ void llama_kv_cache::clear(bool data) { + } + + if (data) { +- for (auto & [_, buf] : ctxs_bufs) { +- ggml_backend_buffer_clear(buf.get(), 0); ++ // opencoti F4 M3 Phase 1: if the deferred zero-fill has not happened ++ // yet there is nothing to clear — v_cells.reset() above already ++ // satisfies the logical "data cleared" state. Don't commit RSS just ++ // to zero pages that have never been written. ++ if (cleared) { ++ for (auto & [_, buf] : ctxs_bufs) { ++ ggml_backend_buffer_clear(buf.get(), 0); ++ } + } + } + } +@@ -630,6 +673,9 @@ llama_memory_context_ptr llama_kv_cache::init_batch( + bool embd_all) { + GGML_UNUSED(embd_all); + ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ ensure_cleared(); ++ + do { + balloc.split_reset(); + +@@ -674,6 +720,12 @@ llama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool + } + + llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector & ubatches) { ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // The iswa wrapper calls prepare() directly on each inner kv_cache without ++ // going through init_batch(), so the ensure_cleared() in init_batch alone ++ // is not enough. Calling here covers both paths and is idempotent. ++ ensure_cleared(); ++ + llama_kv_cache::slot_info_vec_t res; + + struct state_t { +@@ -744,6 +796,15 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co + + auto * sched = lctx->get_sched(); + ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // update() can be reached without init_batch first (e.g. update-only paths ++ // for cross-stream copy or RoPE shift). When that happens with the buffer ++ // still deferred we have nothing to copy or shift — make the dependency ++ // explicit so the tensor->data dereferences below are safe. ++ if (!sc_info.empty() || do_shift) { ++ ensure_cleared(); ++ } ++ + if (!sc_info.empty()) { + assert(n_stream > 1 && "stream copy should never happen with a single stream"); + +@@ -1900,6 +1961,11 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama + + GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size())); + ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // state_read writes the saved KV blob into our tensors, which requires ++ // backing storage. Force the deferred allocation before we read. ++ ensure_cleared(); ++ + uint32_t n_stream_cur; + io.read(&n_stream_cur, sizeof(n_stream_cur)); + if (n_stream_cur != n_stream) { +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -152,6 +152,16 @@ public: + + bool get_has_shift() const; + ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // Idempotent: zeroes the KV-cache backend buffers on first call. ++ // The buffers are malloc'd eagerly at construction (so the scheduler ++ // can resolve tensor->buffer at reserve time) but NOT cleared — on ++ // Linux, malloc reserves address space without committing pages, so ++ // RSS stays low until something writes. This method does the memset ++ // that commits the pages. Called from inference entry points ++ // (init_batch, prepare, update, state_read). ++ void ensure_cleared(); ++ + ggml_type type_k() const; + ggml_type type_v() const; + +@@ -256,6 +266,13 @@ private: + // ggml contexts for the KV cache along with the allocated backend buffers: + std::vector> ctxs_bufs; + ++ // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md ++ // True once the backend buffers in ctxs_bufs have been zero-filled. ++ // The constructor sets this true for no_alloc models (whose dummy ++ // buffers are size 0 — clearing is a no-op) and false otherwise. The ++ // first ensure_cleared() call zeroes the buffers and flips this true. ++ bool cleared = false; ++ + // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot()) + // note: this is not part of the KV state and it's only used to speed-up the find_slot() method + std::vector v_heads; diff --git a/patches/0002-lazy-slot-context-grow.patch b/patches/0002-lazy-slot-context-grow.patch new file mode 100644 index 0000000000000000000000000000000000000000..63799205a931376f4df9d8921b3ecd6724f91026 --- /dev/null +++ b/patches/0002-lazy-slot-context-grow.patch @@ -0,0 +1,596 @@ +From: opencoti +Subject: [PATCH 2/N] lazy slot-context allocation — Phase 2 (grow on demand + partial zero-fill) + +Phase 2 of the F4 M3 lazy slot-context rollout. Builds on Phase 1 +(deferred zero-fill) by adding a soft allocation cap that +find_slot honors, a grow-on-demand path that bumps the cap when +real prompts need more cells, and a partial-range zero-fill so +RSS commits scale with actual workload — not with the configured +n_ctx_slot × slot_count product. + +The end-state for the production embedder +(qwen3-0.6b, --parallel 2 --ctx-size 16384, CPU mode): +- With --slot-initial-ctx 0 (the default) Phase 2 is a no-op + layered on Phase 1: cap == n_ctx_seq from construction; full + buffer cleared on first batch (matches Phase 1 RSS profile). +- With --slot-initial-ctx 4096 (the recommended embedder + setting) the warm RSS scales with whatever cells real prompts + have touched. A run that only ever embeds 8–56-token requests + commits ~4096 × per-cell bytes, not the full 16384. + +The patch threads `slot_initial_ctx` through: +- public API: llama_context_params (include/llama.h) +- internal: llama_cparams (src/llama-cparams.h) +- ctor chain: llama_context::ctor (src/llama-context.cpp) + llama_kv_cache::ctor (src/llama-kv-cache.{h,cpp}) + llama_kv_cache_iswa::ctor (src/llama-kv-cache-iswa.{h,cpp}) + llama_model::create_memory call sites (src/llama-model.cpp) +- hybrid mem: src/llama-memory-hybrid.cpp + -iswa.cpp pass 0 + (kept on Phase-1 behavior — recurrent/hybrid models + are not the F4 embedder target) +- CLI: common_params.slot_initial_ctx + --slot-initial-ctx + (common/common.{h,cpp}, common/arg.cpp) + +Mechanism (inside llama_kv_cache): +- kv_size_max: hard cap (== constructor's kv_size). +- kv_size_alloc: soft cap; cells [0, kv_size_alloc) visible to + find_slot. Starts at min(slot_initial_ctx, + kv_size_max) — Phase 1 default of 0 collapses + to kv_size_max so find_slot sees everything. +- n_cells_cleared: half-open range of cells already zero-filled. + Phase 1's `bool cleared` is replaced; the cells + beyond n_cells_cleared have malloc'd address + space but uncommitted pages. +- ensure_cleared(up_to): + Partial memset of cells [n_cells_cleared, up_to) on each + layer's K and V tensor across every stream, using + ggml_backend_tensor_memset (host-or-device dispatch). The + common inference path passes up_to=0 which expands to the + current soft cap; state_read passes up_to=kv_size_max for + full coverage on the deserialize path. +- ensure_capacity(cells_needed): + Geometric grow (next-pow2, n_pad-aligned) of kv_size_alloc + up to kv_size_max. Monotonic — never shrinks (Phase 3 adds + shrink-on-idle). +- prepare(ubatches): + When find_slot fails inside the current soft cap, retries + after ensure_capacity(used_max + ubatch.n_tokens) + + ensure_cleared(). Bounded by kv_size_max; a genuine + overflow at the hard cap propagates up unchanged. + +Bench: docs/decisions/0001-lazy-slot-context.md §6.2 Phase 2 row. +Milestone: F4 M3 — see docs/features/memory_embedder.md +Design: docs/decisions/0001-lazy-slot-context.md +Upstreaming: deferred; propose to llamafile + llama.cpp + maintainers after F4 M3's full phase-set ships and bench + numbers are public. + +--- +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1361,6 +1361,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.kv_unified = value; + } + ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL})); ++ add_opt(common_arg( ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ {"--slot-initial-ctx"}, "N", ++ string_format("initial KV-cell allocation per sequence (default: %d, 0 = full n_ctx_seq, " ++ "positive values cap initial allocation and enable grow-on-demand)", ++ params.slot_initial_ctx), ++ [](common_params & params, int value) { ++ params.slot_initial_ctx = value; ++ } ++ ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL})); + add_opt(common_arg( + {"--cache-idle-slots"}, + {"--no-cache-idle-slots"}, +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1628,6 +1628,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & + + cparams.n_ctx = params.n_ctx; + cparams.n_seq_max = params.n_parallel; ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_initial_ctx = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -425,6 +425,11 @@ struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata); + struct common_params { + int32_t n_predict = -1; // max. number of new tokens to predict, -1 == no limit + int32_t n_ctx = 0; // context size, 0 == context the model was trained with ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Initial KV-cell allocation per sequence. 0 = full n_ctx_seq allocation ++ // (Phase 1 behavior — only zero-fill is deferred). Positive values cap ++ // the initial allocation and enable grow-on-demand up to n_ctx_seq. ++ int32_t slot_initial_ctx = 0; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -338,6 +338,14 @@ extern "C" { + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Initial KV-cell allocation per sequence. 0 = behave like Phase 1 ++ // (allocate the full n_ctx_seq up front, only zero-fill is deferred). ++ // Positive values cap the initial allocation at min(slot_initial_ctx, ++ // n_ctx_seq); subsequent prompts that exceed this cap trigger a grow ++ // up to next_pow2 (still capped at n_ctx_seq). Reduces warm-state RSS ++ // when typical prompts are much shorter than n_ctx_seq. ++ uint32_t slot_initial_ctx; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -58,6 +58,9 @@ llama_context::llama_context( + cparams.n_rs_seq = 0; + } + ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_initial_ctx = params.slot_initial_ctx; ++ + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; + cparams.yarn_ext_factor = params.yarn_ext_factor >= 0.0f ? params.yarn_ext_factor : hparams.yarn_ext_factor; +@@ -3346,6 +3349,7 @@ llama_context_params llama_context_default_params() { + /*.n_batch =*/ 2048, + /*.n_ubatch =*/ 512, + /*.n_seq_max =*/ 1, ++ /*.slot_initial_ctx =*/ 0, + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -12,6 +12,10 @@ struct llama_cparams { + uint32_t n_batch; + uint32_t n_ubatch; + uint32_t n_seq_max; ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // 0 = full n_ctx_seq allocation. Positive value caps the initial KV-cell ++ // allocation per sequence and enables grow-on-demand up to n_ctx_seq. ++ uint32_t slot_initial_ctx; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -20,6 +20,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + bool swa_full, + bool unified, + uint32_t kv_size, ++ uint32_t kv_size_initial, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +@@ -61,14 +62,14 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + + kv_base = std::make_unique( + model, type_k, type_v, +- v_trans, offload, unified, size_base, n_seq_max, n_pad, ++ v_trans, offload, unified, size_base, kv_size_initial, n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + + kv_swa = std::make_unique( + model, type_k, type_v, +- v_trans, offload, unified, size_swa, n_seq_max, n_pad, ++ v_trans, offload, unified, size_swa, kv_size_initial, n_seq_max, n_pad, + hparams.n_swa, hparams.swa_type, filter_swa, reuse); + } + +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -22,6 +22,9 @@ public: + bool swa_full, + bool unified, + uint32_t kv_size, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Forwarded to both kv_base and kv_swa. 0 = full kv_size. ++ uint32_t kv_size_initial, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -85,6 +85,7 @@ llama_kv_cache::llama_kv_cache( + bool offload, + bool unified, + uint32_t kv_size, ++ uint32_t kv_size_initial, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -96,6 +97,24 @@ llama_kv_cache::llama_kv_cache( + + GGML_ASSERT(kv_size % n_pad == 0); + ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Initialize the hard cap and the soft cap. kv_size_initial==0 (the ++ // legacy / Phase-1 default) means "no soft cap" so the whole kv_size ++ // is visible to find_slot. A positive value is clamped to [n_pad, ++ // kv_size] and rounded up to the next n_pad multiple. ++ kv_size_max = kv_size; ++ if (kv_size_initial == 0) { ++ kv_size_alloc = kv_size_max; ++ } else { ++ uint32_t cap = std::min(kv_size_initial, kv_size_max); ++ cap = std::max(cap, n_pad); ++ cap = ((cap + n_pad - 1) / n_pad) * n_pad; ++ cap = std::min(cap, kv_size_max); ++ kv_size_alloc = cap; ++ LLAMA_LOG_INFO("%s: slot_initial_ctx = %u cells (cap = %u)\n", ++ __func__, kv_size_alloc, kv_size_max); ++ } ++ + const uint32_t n_layer_kv = hparams.n_layer_kv(); + + // define a comparator for the buft -> ctx map to ensure that the order is well-defined: +@@ -287,7 +306,9 @@ llama_kv_cache::llama_kv_cache( + } + + if (model.hparams.no_alloc) { +- cleared = true; ++ // opencoti F4 M3 Phase 1+2 — dummy buffers are size 0; treat all ++ // cells as "already cleared" so ensure_cleared is a no-op. ++ n_cells_cleared = kv_size_max; + } else { + LLAMA_LOG_INFO("%s: KV cache zero-fill deferred until first batch\n", __func__); + } +@@ -348,20 +369,81 @@ llama_kv_cache::llama_kv_cache( + debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0; + } + +-// opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md +-void llama_kv_cache::ensure_cleared() { +- if (cleared) { ++// opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md ++void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) { ++ // up_to_cells == 0 means "clear up to the current soft cap". Callers ++ // that need full coverage (state_read) pass kv_size_max. ++ const uint32_t target = up_to_cells == 0 ++ ? kv_size_alloc ++ : std::min(up_to_cells, kv_size_max); ++ ++ if (n_cells_cleared >= target) { + return; + } + +- for (auto & [_, buf] : ctxs_bufs) { +- LLAMA_LOG_INFO("%s: %10s KV buffer zero-fill = %8.2f MiB (deferred)\n", __func__, +- ggml_backend_buffer_name(buf.get()), +- ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0); +- ggml_backend_buffer_clear(buf.get(), 0); ++ const uint32_t a = n_cells_cleared; ++ const uint32_t b = target; ++ ++ // Per-layer, per-stream partial memset of the K and V tensors. ++ // ggml_backend_tensor_memset handles the host/device dispatch — for the ++ // CPU backend (the only one the embedder uses today) this is a direct ++ // memset on the malloc'd buffer; for offloaded buffers it's a backend ++ // op. Either way only bytes for cells [a, b) get touched, so on Linux ++ // only those pages get committed to RSS. ++ for (const auto & layer : layers) { ++ for (uint32_t st = 0; st < n_stream; ++st) { ++ if (layer.k) { ++ const size_t off = (size_t) st * (size_t) layer.k->nb[2] ++ + (size_t) a * (size_t) layer.k->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) layer.k->nb[1]; ++ ggml_backend_tensor_memset(layer.k, 0, off, sz); ++ } ++ if (layer.v) { ++ const size_t off = (size_t) st * (size_t) layer.v->nb[2] ++ + (size_t) a * (size_t) layer.v->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) layer.v->nb[1]; ++ ggml_backend_tensor_memset(layer.v, 0, off, sz); ++ } ++ } ++ } ++ ++ LLAMA_LOG_INFO("%s: cleared KV cells [%u, %u) of %u (%s)\n", ++ __func__, a, b, kv_size_max, ++ up_to_cells == 0 ? "soft cap" : "full"); ++ ++ n_cells_cleared = b; ++} ++ ++// opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) { ++ if (cells_needed <= kv_size_alloc) { ++ return true; ++ } ++ ++ if (kv_size_alloc >= kv_size_max) { ++ return false; ++ } ++ ++ // Grow geometrically (next power-of-two) and align up to n_pad. The ++ // pow2 ramp avoids re-growing every few tokens for long prompts; n_pad ++ // alignment preserves the constructor's GGML_ASSERT(kv_size % n_pad == 0) ++ // invariant for find_slot. ++ uint32_t new_cap = std::max(kv_size_alloc, n_pad); ++ while (new_cap < cells_needed && new_cap < kv_size_max) { ++ new_cap *= 2; + } ++ new_cap = ((new_cap + n_pad - 1) / n_pad) * n_pad; ++ new_cap = std::min(new_cap, kv_size_max); + +- cleared = true; ++ if (new_cap == kv_size_alloc) { ++ return cells_needed <= kv_size_alloc; ++ } ++ ++ LLAMA_LOG_INFO("%s: grow soft cap %u -> %u (need %u, max %u)\n", ++ __func__, kv_size_alloc, new_cap, cells_needed, kv_size_max); ++ ++ kv_size_alloc = new_cap; ++ return cells_needed <= kv_size_alloc; + } + + void llama_kv_cache::clear(bool data) { +@@ -371,14 +453,16 @@ void llama_kv_cache::clear(bool data) { + } + + if (data) { +- // opencoti F4 M3 Phase 1: if the deferred zero-fill has not happened +- // yet there is nothing to clear — v_cells.reset() above already +- // satisfies the logical "data cleared" state. Don't commit RSS just +- // to zero pages that have never been written. +- if (cleared) { +- for (auto & [_, buf] : ctxs_bufs) { +- ggml_backend_buffer_clear(buf.get(), 0); +- } ++ // opencoti F4 M3 Phase 1+2: if no zero-fill has happened yet there ++ // is nothing to clear — v_cells.reset() above already satisfies the ++ // logical "data cleared" state. Don't commit RSS just to zero pages ++ // that have never been written. When a partial clear is already on ++ // the books, re-issue it against the current soft cap so any cells ++ // touched between the first ensure_cleared and now are re-zeroed. ++ if (n_cells_cleared > 0) { ++ const uint32_t b = n_cells_cleared; ++ n_cells_cleared = 0; ++ ensure_cleared(b); + } + } + } +@@ -743,10 +827,28 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector + // better to start searching from the beginning of the cache, hoping to fill it + if (head_cur > cells.get_used() + 2*n_tokens) { + head_cur = 0; + } + +- if (n_tokens > cells.size()) { +- LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size()); ++ if (n_tokens > cap) { ++ // not necessarily fatal — prepare() may grow the cap and retry + return { }; + } + +@@ -999,8 +1107,8 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, + const uint32_t n_test = cont ? n_tokens : 1; + + while (true) { +- if (head_cur + n_test > cells.size()) { +- n_tested += cells.size() - head_cur; ++ if (head_cur + n_test > cap) { ++ n_tested += cap - head_cur; + head_cur = 0; + continue; + } +@@ -1058,7 +1166,7 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, + res.idxs[s].clear(); + } + +- if (n_tested >= cells.size()) { ++ if (n_tested >= cap) { + //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens); + return { }; + } +@@ -1961,10 +2069,13 @@ void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama + + GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size())); + +- // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md +- // state_read writes the saved KV blob into our tensors, which requires +- // backing storage. Force the deferred allocation before we read. +- ensure_cleared(); ++ // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md ++ // state_read writes the saved KV blob into our tensors at arbitrary ++ // offsets, so we cannot rely on the soft cap. Force a full-range ++ // zero-fill before we read — this commits all KV pages but only on ++ // the explicit deserialize path. ++ ensure_capacity(kv_size_max); ++ ensure_cleared(kv_size_max); + + uint32_t n_stream_cur; + io.read(&n_stream_cur, sizeof(n_stream_cur)); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -101,6 +101,11 @@ public: + bool offload, + bool unified, + uint32_t kv_size, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Initial allocation cap. 0 means the same as kv_size ++ // (Phase 1 behavior). Otherwise clamped to [n_pad, kv_size] ++ // and rounded up to the next n_pad multiple. ++ uint32_t kv_size_initial, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -152,15 +157,28 @@ public: + + bool get_has_shift() const; + +- // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md +- // Idempotent: zeroes the KV-cache backend buffers on first call. +- // The buffers are malloc'd eagerly at construction (so the scheduler +- // can resolve tensor->buffer at reserve time) but NOT cleared — on +- // Linux, malloc reserves address space without committing pages, so +- // RSS stays low until something writes. This method does the memset +- // that commits the pages. Called from inference entry points +- // (init_batch, prepare, update, state_read). +- void ensure_cleared(); ++ // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md ++ // Idempotent. Zeroes cells [n_cells_cleared, up_to_cells) of every KV ++ // layer's K and V tensors across every stream. The buffers are malloc'd ++ // eagerly at construction (so the scheduler can resolve tensor->buffer ++ // at reserve time) but their pages are not committed to RSS until they ++ // are first written. Partial memset (via ggml_backend_tensor_memset) ++ // commits only the requested range — that is the Phase 2 RSS win. ++ // ++ // up_to_cells == 0 means "use the current soft cap kv_size_alloc" ++ // (the normal inference path). Pass kv_size_max for paths that need ++ // full coverage (state_read). ++ void ensure_cleared(uint32_t up_to_cells = 0); ++ ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Grow the soft allocation cap. Returns true iff the cap covers ++ // cells_needed after the grow. Bound by kv_size_max. Idempotent at ++ // the cap. ++ bool ensure_capacity(uint32_t cells_needed); ++ ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // The current soft cap (cells visible to find_slot). ++ uint32_t get_kv_size_alloc() const { return kv_size_alloc; } + + ggml_type type_k() const; + ggml_type type_v() const; +@@ -266,12 +284,22 @@ private: + // ggml contexts for the KV cache along with the allocated backend buffers: + std::vector> ctxs_bufs; + +- // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md +- // True once the backend buffers in ctxs_bufs have been zero-filled. +- // The constructor sets this true for no_alloc models (whose dummy +- // buffers are size 0 — clearing is a no-op) and false otherwise. The +- // first ensure_cleared() call zeroes the buffers and flips this true. +- bool cleared = false; ++ // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md ++ // Cells in the half-open range [0, n_cells_cleared) are guaranteed to ++ // be zeroed across every layer's K and V tensors and every stream. ++ // ensure_cleared(up_to) grows this monotonically up to kv_size_alloc ++ // (or kv_size_max for the explicit-full call). The constructor sets ++ // this to kv_size_max for no_alloc models (whose dummy buffers are ++ // size 0 — clearing is a no-op) and 0 otherwise. ++ uint32_t n_cells_cleared = 0; ++ ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // kv_size_max: hard cap on cells (set from constructor's kv_size). ++ // kv_size_alloc: current soft cap. find_slot only scans cells [0, kv_size_alloc). ++ // Grows via ensure_capacity, never shrinks (Phase 2; Phase 3 ++ // adds shrink-on-idle). ++ uint32_t kv_size_max = 0; ++ uint32_t kv_size_alloc = 0; + + // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot()) + // note: this is not part of the KV state and it's only used to speed-up the find_slot() method +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -40,6 +40,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + swa_full, + unified, + kv_size, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Hybrid-iswa does not yet thread slot_initial_ctx; pass 0 for ++ // Phase 1 backward-compat (full eager allocation). ++ 0, + n_seq_max, + n_ubatch, + n_pad, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -39,6 +39,10 @@ llama_memory_hybrid::llama_memory_hybrid( + offload, + unified, + kv_size, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Hybrid memory does not yet thread slot_initial_ctx; pass 0 for ++ // Phase 1 backward-compat (full eager allocation). ++ 0, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2073,6 +2073,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_initial_ctx, + cparams.n_seq_max, + cparams.n_ubatch, + 1, +@@ -2089,6 +2091,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_initial_ctx, + cparams.n_seq_max, + 1, + hparams.n_swa, diff --git a/patches/0003-arg-assertion-bypass.patch b/patches/0003-arg-assertion-bypass.patch new file mode 100644 index 0000000000000000000000000000000000000000..06582883f76d2905846145002095cd51687367b0 --- /dev/null +++ b/patches/0003-arg-assertion-bypass.patch @@ -0,0 +1,46 @@ +From: opencoti +Subject: [PATCH 3/N] bypass upstream `params.n_gpu_layers < 0` assertions + +Upstream llama.cpp pin 5e9c63546 has TWO `GGML_ASSERT(n_gpu_layers < 0)` +calls in common/arg.cpp: + +1. At the `-ngl` option registration site (params.n_gpu_layers). +2. At the `-ngld` (draft) option registration site + (params.speculative.n_gpu_layers). + +Both fire on every `common_params_parser_init` call regardless of args, +making the patched binary unable to start in `--server`, `--cli`, or +`--chat` mode. Mozilla-Ocho v1.4.0 (pre-assertion) was unaffected. + +Each assertion is a "display-format guard" for the next line, which +maps `n_gpu_layers == -1` to "auto" and anything else to "all" in the +help text. The fix is to either extend the format to handle non- +negative defaults, or to drop the assertion. We drop it — the +display "all" for a positive default is a minor cosmetic issue, +not a correctness problem. + +This is an upstream-bug-bypass patch, not opencoti scope. To be +proposed upstream once we surface a minimal reproducer. + +Bench: closes the F4 M3 bench path that requires booting the +patched binary in --server mode. +Milestone: F4 M3 — see docs/features/memory_embedder.md +Upstreaming: TBD — propose to llama.cpp once a minimal repro is + in hand. Likely already fixed in newer upstream commits. + +--- +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -2362,7 +2362,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + } + } + ).set_env("LLAMA_ARG_N_CPU_MOE")); +- GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0 ++ // opencoti F4 M3 patch 0003 — see docs/decisions/0001-lazy-slot-context.md ++ // Upstream assertion fires on every common_params_parser_init invocation ++ // (display-format guard for the next line). Dropped — display says "all" ++ // for non-negative defaults, which is cosmetic, not a correctness bug. + add_opt(common_arg( + {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N", + string_format("max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: %s)", params.n_gpu_layers == -1 ? "auto" : "all"), diff --git a/patches/0004-cuda-std-cpp17.patch b/patches/0004-cuda-std-cpp17.patch new file mode 100644 index 0000000000000000000000000000000000000000..8611e8967d7f774232eb9911ad99df84a2699178 --- /dev/null +++ b/patches/0004-cuda-std-cpp17.patch @@ -0,0 +1,50 @@ +From: opencoti +Subject: [PATCH 4/N] add -std=c++17 to ggml-cuda NVCC flags + +Mozilla-Ocho's `llamafile/cuda.sh` builds the `ggml-cuda.so` DSO that +the llamafile runtime loads when the user passes `-ngl `. The +ggml-cuda sources have moved to C++17 features (`std::is_same_v`, fold +expressions, structured bindings), but cuda.sh's COMMON_FLAGS do not +pass `-std=c++17` to nvcc. + +For CUDA 11.x and older 12.x toolkits this worked because nvcc +defaulted to C++14 and the sources were C++14-clean. Newer CUDA +toolkits ship CCCL that requires C++17 explicitly via a #error +guard, so building against CUDA 12.6+ (and definitely CUDA 13.x) +fails with `error: namespace "std" has no member "is_same_v"` and +~30 follow-on errors per .cu translation unit. + +This patch adds `-std=c++17` to COMMON_FLAGS so cuda.sh builds +against any reasonable CUDA toolkit. The host compiler is already +on C++23 via cosmocc, so the host-flag side is fine; this is +purely the nvcc device-flag side. + +Bench: closes the GPU validation path. Without this, the patched +llamafile cannot load ggml-cuda.so on solidPC (and on any host +where CUDA 12.6+ is the local toolkit), and the binary silently +falls back to CPU even with `-ngl all`. +Milestone: F4 M3 GPU validation (task #113) — see + docs/decisions/0001-lazy-slot-context.md and + docs/features/memory_embedder.md +Upstreaming: TBD — propose to Mozilla-Ocho once they decide on a + minimum-supported CUDA toolkit. Currently their build matrix is + unclear about whether CUDA 13.x is expected to work; this patch + is the minimal fix that does not regress older toolkits. + +--- +diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh +--- a/llamafile/cuda.sh ++++ b/llamafile/cuda.sh +@@ -232,7 +232,12 @@ if [ "$COMPRESS" = "1" ]; then + fi + + # NVCC compiler flags ++# opencoti note: -std=c++17 added because the ggml-cuda .cu sources use ++# C++17 features (std::is_same_v, structured bindings, fold expressions). ++# Newer CUDA toolkits (12.x+) need this explicitly; older toolkits did C++14 ++# by default. Both work with c++17. + COMMON_FLAGS="\ ++ -std=c++17 \ + --use_fast_math \ + --extended-lambda \ + $EXTRA_INCLUDES \ diff --git a/patches/0005-lazy-slot-context-shrink.patch b/patches/0005-lazy-slot-context-shrink.patch new file mode 100644 index 0000000000000000000000000000000000000000..330ce71d76df179af6b4cd03defd8f1973ab42f5 --- /dev/null +++ b/patches/0005-lazy-slot-context-shrink.patch @@ -0,0 +1,582 @@ +From: opencoti +Subject: [PATCH 5/N] lazy slot-context allocation — Phase 3 (shrink on idle) + +Phase 2 added a soft cap on per-stream KV cells (kv_size_alloc <= kv_size_max) +and grow-on-demand via ensure_capacity. The soft cap never shrinks, so once a +single long prompt has caused it to grow, the embedder's RSS stays at the +grown size for the lifetime of the process. For an embedder running +`--parallel 2 --ctx-size 16384` with typical prompts well under 4K cells, +this means the headline Phase-2 win evaporates after the first long-context +request. + +Phase 3 plugs that hole. When an idle threshold has elapsed since the last +apply_ubatch: + + 1. Cells in the range [kv_size_shrink_target, kv_size_alloc) are evicted + across every stream via cells.rm(i). The server-context layer's + prompt cache keeps cells alive between embed requests for reuse; + decommitting their backing pages without first evicting would leave + the prompt cache pointing at zeroed memory and corrupt subsequent + re-uses. The eviction makes the shrink unconditional. + + 2. The pages backing cells [target, current_alloc) of every layer's K + and V tensors are returned to the kernel via + madvise(addr, len, MADV_DONTNEED). + + 3. n_cells_cleared is rolled back to the shrink target so the next + ensure_cleared() partial-memset re-zeros only what comes back. + + 4. kv_size_alloc is reset to the shrink target so the next request can + grow again from the initial cap. + +Why madvise(MADV_DONTNEED) and not posix_madvise(POSIX_MADV_DONTNEED): +glibc on Linux maps posix_madvise(POSIX_MADV_DONTNEED) to a no-op (see +glibc sysdeps/posix/posix_madvise.c — the standard explicitly forbids +"actively destroying" pages, which is what we need). Only the Linux- +specific madvise(MADV_DONTNEED) actually decommits anonymous pages and +returns RSS to the kernel. We declared MADV_DONTNEED with the canonical +Linux value (4) under #ifndef and extern-declared madvise() so cosmocc's +default feature set (which does not define _GNU_SOURCE and so does not +expose madvise() through sys/mman.h) builds cleanly. + +The shrink check runs at the top of init_batch() and prepare() — both +entry points to actual KV work. It is gated by: + - slot_shrink_idle_ms_ > 0 (feature enabled) + - kv_size_alloc > kv_size_shrink_target (something to shrink) + - last_active_ms_ != 0 AND (now - last_active_ms_) >= idle threshold + +Surface (CLI): + --slot-shrink-idle-ms N (default 0 = disabled; positive enables) + env LLAMA_ARG_SLOT_SHRINK_IDLE_MS + +Surface (C API): + llama_context_params::slot_shrink_idle_ms (uint32_t, default 0) + +Plumbed through llama_cparams -> llama_kv_cache ctor -> +(llama_kv_cache_iswa ctor for SWA models) -> llama-model.cpp call sites. +Backward-compat: llama-memory-hybrid{,-iswa}.cpp pass 0 (no shrink), the +same way they already pass 0 for kv_size_initial. + +Bench (CPU, --gpu disable so KV lives in host pages): the embedder +configured `--parallel 2 --ctx-size 16384 --pooling last +--slot-initial-ctx 256 --slot-shrink-idle-ms 30000` starts at 719 MB +RSS, grows to 1720 MB after a long-prompt corpus pass (soft cap 256 -> +2048), and after the 30 s idle drops to 1329 MB on the next short embed. +391.7 MB returned to the kernel — measured by /proc//status VmRSS. +Quality (cosine min vs M2 baseline) stays at 0.999797 across the 48-prompt +corpus. + +Milestone: F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md +Upstreaming: TBD — kv_cache idle-shrink is a generally-useful concept; the + cell-eviction + madvise(MADV_DONTNEED) approach is not opencoti-specific. + Proposal candidate after F4 closes. + +--- +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1371,6 +1371,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.slot_initial_ctx = value; + } + ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL})); ++ add_opt(common_arg( ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ {"--slot-shrink-idle-ms"}, "N", ++ string_format("idle threshold (ms) after which a grown KV soft cap is shrunk back to " ++ "--slot-initial-ctx and the over-cap pages are decommitted via posix_madvise " ++ "(default: %d, 0 disables; requires --slot-initial-ctx > 0)", ++ params.slot_shrink_idle_ms), ++ [](common_params & params, int value) { ++ params.slot_shrink_idle_ms = value; ++ } ++ ).set_env("LLAMA_ARG_SLOT_SHRINK_IDLE_MS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL})); + add_opt(common_arg( + {"--cache-idle-slots"}, + {"--no-cache-idle-slots"}, +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1630,6 +1630,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.n_seq_max = params.n_parallel; + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_initial_ctx = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -430,6 +430,11 @@ struct common_params { + // (Phase 1 behavior — only zero-fill is deferred). Positive values cap + // the initial allocation and enable grow-on-demand up to n_ctx_seq. + int32_t slot_initial_ctx = 0; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Idle threshold (ms) after which a grown soft cap is shrunk back to ++ // slot_initial_ctx and the over-cap pages are decommitted via ++ // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior). ++ int32_t slot_shrink_idle_ms = 0; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -346,6 +346,13 @@ extern "C" { + // up to next_pow2 (still capped at n_ctx_seq). Reduces warm-state RSS + // when typical prompts are much shorter than n_ctx_seq. + uint32_t slot_initial_ctx; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Idle threshold after which a grown soft cap is shrunk back to ++ // slot_initial_ctx and the pages beyond it are returned to the OS via ++ // posix_madvise(POSIX_MADV_DONTNEED). 0 disables shrink-on-idle ++ // (Phase 2 behavior — soft cap grows monotonically). Requires ++ // slot_initial_ctx > 0 to have any visible effect. ++ uint32_t slot_shrink_idle_ms; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -60,6 +60,8 @@ llama_context::llama_context( + + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_initial_ctx = params.slot_initial_ctx; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms; + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -3350,6 +3352,7 @@ llama_context_params llama_context_default_params() { + /*.n_ubatch =*/ 512, + /*.n_seq_max =*/ 1, + /*.slot_initial_ctx =*/ 0, ++ /*.slot_shrink_idle_ms =*/ 0, + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -16,6 +16,11 @@ struct llama_cparams { + // 0 = full n_ctx_seq allocation. Positive value caps the initial KV-cell + // allocation per sequence and enables grow-on-demand up to n_ctx_seq. + uint32_t slot_initial_ctx; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // 0 disables shrink-on-idle. Positive value is the idle threshold in ++ // milliseconds: when a grown KV-cache is idle for at least this long, the ++ // soft cap is reset to slot_initial_ctx and the pages beyond are decommitted. ++ uint32_t slot_shrink_idle_ms; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -21,6 +21,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + bool unified, + uint32_t kv_size, + uint32_t kv_size_initial, ++ uint32_t slot_shrink_idle_ms, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +@@ -62,15 +63,15 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + + kv_base = std::make_unique( + model, type_k, type_v, +- v_trans, offload, unified, size_base, kv_size_initial, n_seq_max, n_pad, +- 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); ++ v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, ++ n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + + kv_swa = std::make_unique( + model, type_k, type_v, +- v_trans, offload, unified, size_swa, kv_size_initial, n_seq_max, n_pad, +- hparams.n_swa, hparams.swa_type, filter_swa, reuse); ++ v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, ++ n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse); + } + + void llama_kv_cache_iswa::clear(bool data) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -25,6 +25,9 @@ public: + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + // Forwarded to both kv_base and kv_swa. 0 = full kv_size. + uint32_t kv_size_initial, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Forwarded to both kv_base and kv_swa. 0 = shrink disabled. ++ uint32_t slot_shrink_idle_ms, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -7,12 +7,35 @@ + + #include + #include ++#include + #include + #include + #include + #include + #include + ++// opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++// On Linux, madvise(addr, len, MADV_DONTNEED) is the call that actually ++// decommits pages: the kernel zeroes the page table entries, releases the ++// physical pages, and the next access faults in fresh zero-filled pages. ++// glibc's posix_madvise(POSIX_MADV_DONTNEED) is documented as advisory and ++// is NOT guaranteed to reduce RSS — empirically it is a no-op on the ++// glibc/Debian-11 setup the embedder runs on. We must call madvise() ++// directly. ++// ++// cosmocc's default headers (without _GNU_SOURCE) do not declare madvise() ++// or MADV_DONTNEED, but the syscall exists on every Linux kernel the ++// embedder runs on. Hardcode the Linux ABI value (asm-generic/mman-common.h: ++// `#define MADV_DONTNEED 4`) and forward-declare madvise() to keep ++// _GNU_SOURCE out of the global include. provides ++// sysconf(_SC_PAGESIZE) for page-boundary math. ++#include ++#include ++#ifndef MADV_DONTNEED ++# define MADV_DONTNEED 4 ++#endif ++extern "C" int madvise(void *addr, size_t length, int advice); ++ + static bool ggml_is_power_of_2(int n) { + return (n & (n - 1)) == 0; + } +@@ -86,6 +109,7 @@ llama_kv_cache::llama_kv_cache( + bool unified, + uint32_t kv_size, + uint32_t kv_size_initial, ++ uint32_t slot_shrink_idle_ms, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -115,6 +139,18 @@ llama_kv_cache::llama_kv_cache( + __func__, kv_size_alloc, kv_size_max); + } + ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // The shrink target is the initial cap we just computed. last_active_ms_ ++ // starts at 0 (epoch) so a long pause before the first request still ++ // shrinks (correct behavior — nothing to lose). ++ kv_size_shrink_target = kv_size_alloc; ++ slot_shrink_idle_ms_ = slot_shrink_idle_ms; ++ last_active_ms_ = 0; ++ if (slot_shrink_idle_ms_ > 0 && kv_size_initial > 0 && kv_size_initial < kv_size_max) { ++ LLAMA_LOG_INFO("%s: slot_shrink_idle_ms = %u (target cap = %u)\n", ++ __func__, slot_shrink_idle_ms_, kv_size_shrink_target); ++ } ++ + const uint32_t n_layer_kv = hparams.n_layer_kv(); + + // define a comparator for the buft -> ctx map to ensure that the order is well-defined: +@@ -446,6 +482,151 @@ bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) { + return cells_needed <= kv_size_alloc; + } + ++// opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++// Decommit pages of every layer's K/V tensors in the byte-range that ++// corresponds to cells [from_cells, to_cells). Returns those pages to the OS ++// — RSS drops by the size of the range, and the next access to those pages ++// will fault them back in as fresh zero-filled pages. ++// ++// Only safe to call after kv_size_alloc has been reset to from_cells (or ++// lower) and n_cells_cleared has been reset accordingly, so the bookkeeping ++// stays in sync with what's actually backed by physical memory. ++// ++// The decommit is per-stream because the KV tensors are 3-D ++// [n_embd, kv_size, n_stream]; cells [a, b) within stream s occupy a ++// contiguous byte-range whose start address is (base + s*nb[2] + a*nb[1]) ++// and length is (b-a)*nb[1]. posix_madvise is page-grained — bytes that ++// don't cover a full page are silently kept resident, which is fine for ++// the cells that straddle page boundaries. ++static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream, ++ uint32_t from_cells, uint32_t to_cells) { ++ if (!t || to_cells <= from_cells) { ++ return; ++ } ++ ggml_backend_buffer_t buf = t->buffer; ++ if (!buf) { ++ return; ++ } ++ // Only the host (CPU) backend's buffer is malloc'd memory we can ++ // decommit via madvise. For device buffers (CUDA/Metal/Vulkan) the K/V ++ // cache lives in device memory; skip — there are no host-side pages ++ // to return. This means Phase 3's RSS-shrink win applies to host-mode ++ // (e.g. embedder with -ngl 0); GPU-mode binaries still benefit from ++ // the soft-cap reset but RSS doesn't move. ++ if (!ggml_backend_buffer_is_host(buf)) { ++ return; ++ } ++ void * base = ggml_backend_buffer_get_base(buf); ++ if (!base) { ++ return; ++ } ++ const size_t stride_stream = (size_t) t->nb[2]; ++ const size_t stride_cell = (size_t) t->nb[1]; ++ const size_t off_in_buf = (size_t) ((char *) t->data - (char *) base); ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ const size_t off = off_in_buf ++ + (size_t) s * stride_stream ++ + (size_t) from_cells * stride_cell; ++ const size_t sz = (size_t) (to_cells - from_cells) * stride_cell; ++ // posix_madvise wants the address — work in absolute terms. ++ void * addr = (char *) base + off; ++ // Round up to a page boundary on the low side, round down on the ++ // high side, so we only decommit pages we fully own. The cell-aligned ++ // address is unlikely to be page-aligned in general. ++ const size_t page = (size_t) sysconf(_SC_PAGESIZE); ++ if (page == 0 || page == (size_t) -1) { ++ return; ++ } ++ uintptr_t lo = (uintptr_t) addr; ++ uintptr_t hi = lo + sz; ++ uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1); ++ uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1); ++ if (hi_aligned <= lo_aligned) { ++ continue; ++ } ++ // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with ++ // errno set). We treat any failure as advisory — the RSS win may ++ // be skipped, but correctness is unaffected (the K/V buffer is ++ // still mapped, just possibly still resident). ++ (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned), ++ MADV_DONTNEED); ++ } ++} ++ ++void llama_kv_cache::shrink_if_idle() { ++ if (slot_shrink_idle_ms_ == 0) { ++ return; // shrink-on-idle disabled ++ } ++ if (kv_size_alloc <= kv_size_shrink_target) { ++ return; // already at (or below) target — nothing to shrink ++ } ++ if (last_active_ms_ == 0) { ++ return; // no apply_ubatch has ever run — leave the initial cap as-is ++ } ++ ++ const uint64_t now = (uint64_t) ++ std::chrono::duration_cast( ++ std::chrono::steady_clock::now().time_since_epoch()).count(); ++ if (now < last_active_ms_) { ++ return; // clock went backwards; refuse to act ++ } ++ if ((now - last_active_ms_) < (uint64_t) slot_shrink_idle_ms_) { ++ return; // not idle long enough ++ } ++ ++ const uint32_t from_cells = kv_size_shrink_target; ++ const uint32_t to_cells = kv_size_alloc; ++ ++ // Evict prompt-cache cell metadata for indices [from_cells, to_cells) ++ // on every stream BEFORE decommitting the pages. The server's prompt ++ // cache (server-context.cpp) keeps cells populated between requests so ++ // a subsequent request with the same prefix can skip re-prefill. If we ++ // decommitted pages without invalidating cell metadata, the server's ++ // next prompt-cache lookup could "match" a prefix whose K/V data is ++ // now decommitted (returns to zero pages on next access) = corruption. ++ // ++ // Semantics: shrink-on-idle is allowed to evict the over-cap portion ++ // of the prompt cache because that is exactly what the user opted into ++ // by setting --slot-shrink-idle-ms > 0. A cache miss on a subsequent ++ // long prompt is the cost; the alternative (cap stays grown forever) ++ // defeats Phase 3 entirely — slots that retain cells between requests ++ // (the default for embed + completion) would never shrink. ++ uint32_t evicted = 0; ++ for (uint32_t st = 0; st < n_stream; ++st) { ++ auto & cells = v_cells[st]; ++ const uint32_t hi = std::min(cells.size(), to_cells); ++ for (uint32_t i = from_cells; i < hi; ++i) { ++ if (!cells.is_empty(i)) { ++ cells.rm(i); ++ ++evicted; ++ } ++ } ++ } ++ ++ LLAMA_LOG_INFO("%s: shrink soft cap %u -> %u (idle %llu ms, evicted %u cells)\n", ++ __func__, to_cells, from_cells, ++ (unsigned long long) (now - last_active_ms_), evicted); ++ ++ for (const auto & layer : layers) { ++ opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells); ++ opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells); ++ } ++ ++ // After decommit, treat cells [from_cells, to_cells) as "uncleared" — ++ // their pages will fault back in as fresh zero-filled pages on next ++ // touch, but n_cells_cleared still recorded them as cleared. Roll back ++ // the cleared mark so a future ensure_cleared() re-zeros them only when ++ // they're actually needed. ++ if (n_cells_cleared > from_cells) { ++ n_cells_cleared = from_cells; ++ } ++ kv_size_alloc = from_cells; ++ ++ // Refresh last_active_ms_ so we don't immediately try again on the very ++ // next init_batch (which would walk the empty layers for nothing). ++ last_active_ms_ = now; ++} ++ + void llama_kv_cache::clear(bool data) { + for (uint32_t s = 0; s < n_stream; ++s) { + v_cells[s].reset(); +@@ -757,6 +938,12 @@ llama_memory_context_ptr llama_kv_cache::init_batch( + bool embd_all) { + GGML_UNUSED(embd_all); + ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Check idle-shrink BEFORE ensure_cleared(): if shrink runs, it lowers ++ // kv_size_alloc and rolls n_cells_cleared back, so ensure_cleared's ++ // soft-cap-bounded clear targets only the (now smaller) initial cap. ++ shrink_if_idle(); ++ + // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md + ensure_cleared(); + +@@ -804,6 +991,12 @@ llama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool + } + + llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector & ubatches) { ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Same reason ensure_cleared() is called here as well as init_batch: ++ // the iswa wrapper bypasses init_batch and calls prepare() directly on ++ // each inner kv_cache. shrink_if_idle is idempotent and cheap. ++ shrink_if_idle(); ++ + // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md + // The iswa wrapper calls prepare() directly on each inner kv_cache without + // going through init_batch(), so the ensure_cleared() in init_batch alone +@@ -1184,6 +1377,13 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, + } + + void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch) { ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Refresh the idle timestamp on every actual write. shrink_if_idle uses ++ // this to gate the shrink path. ++ last_active_ms_ = (uint64_t) ++ std::chrono::duration_cast( ++ std::chrono::steady_clock::now().time_since_epoch()).count(); ++ + // keep track of the max sequence position that we would overwrite with this ubatch + // for non-SWA cache, this would be always empty + llama_seq_id seq_pos_max_rm[LLAMA_MAX_SEQ]; +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -106,6 +106,11 @@ public: + // (Phase 1 behavior). Otherwise clamped to [n_pad, kv_size] + // and rounded up to the next n_pad multiple. + uint32_t kv_size_initial, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Idle threshold (ms) after which the soft cap is shrunk ++ // back to kv_size_initial and the over-cap pages are ++ // decommitted. 0 disables shrink-on-idle. ++ uint32_t slot_shrink_idle_ms, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -180,6 +185,17 @@ public: + // The current soft cap (cells visible to find_slot). + uint32_t get_kv_size_alloc() const { return kv_size_alloc; } + ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // If the soft cap has been grown above kv_size_shrink_target and at least ++ // slot_shrink_idle_ms_ have elapsed since the last apply_ubatch, reset ++ // the soft cap to kv_size_shrink_target and posix_madvise(POSIX_MADV_DONTNEED) ++ // the over-cap pages of every layer's K/V tensors. Idempotent and cheap ++ // when the soft cap is already at the target or shrink-on-idle is disabled. ++ // Called at the top of init_batch and prepare; safe because both happen ++ // before any slot work and the embedder's per-request cells.reset() means ++ // no slot ever holds onto cells across requests. ++ void shrink_if_idle(); ++ + ggml_type type_k() const; + ggml_type type_v() const; + +@@ -296,11 +312,21 @@ private: + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + // kv_size_max: hard cap on cells (set from constructor's kv_size). + // kv_size_alloc: current soft cap. find_slot only scans cells [0, kv_size_alloc). +- // Grows via ensure_capacity, never shrinks (Phase 2; Phase 3 +- // adds shrink-on-idle). ++ // Grows via ensure_capacity. Phase 3 adds shrink-on-idle: ++ // shrink_if_idle() may reset it back to kv_size_shrink_target. + uint32_t kv_size_max = 0; + uint32_t kv_size_alloc = 0; + ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Phase 3 shrink-on-idle state. kv_size_shrink_target is the cap value ++ // the soft cap is reset to on shrink — set once at ctor from ++ // kv_size_initial (or kv_size_max if Phase-1-mode). slot_shrink_idle_ms_ ++ // is the idle threshold in ms; 0 disables shrink. last_active_ms_ is a ++ // monotonic timestamp updated by apply_ubatch. ++ uint32_t kv_size_shrink_target = 0; ++ uint32_t slot_shrink_idle_ms_ = 0; ++ uint64_t last_active_ms_ = 0; ++ + // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot()) + // note: this is not part of the KV state and it's only used to speed-up the find_slot() method + std::vector v_heads; +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -44,6 +44,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + // Hybrid-iswa does not yet thread slot_initial_ctx; pass 0 for + // Phase 1 backward-compat (full eager allocation). + 0, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Hybrid-iswa does not yet thread slot_shrink_idle_ms; pass 0 for ++ // Phase 2 backward-compat (no shrink-on-idle). ++ 0, + n_seq_max, + n_ubatch, + n_pad, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -43,6 +43,10 @@ llama_memory_hybrid::llama_memory_hybrid( + // Hybrid memory does not yet thread slot_initial_ctx; pass 0 for + // Phase 1 backward-compat (full eager allocation). + 0, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Hybrid memory does not yet thread slot_shrink_idle_ms; pass 0 for ++ // Phase 2 backward-compat (no shrink-on-idle). ++ 0, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2075,6 +2075,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.n_ctx_seq, + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_initial_ctx, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_shrink_idle_ms, + cparams.n_seq_max, + cparams.n_ubatch, + 1, +@@ -2093,6 +2095,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.n_ctx_seq, + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_initial_ctx, ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ cparams.slot_shrink_idle_ms, + cparams.n_seq_max, + 1, + hparams.n_swa, diff --git a/patches/0006-kv-reuse-prefix.patch b/patches/0006-kv-reuse-prefix.patch new file mode 100644 index 0000000000000000000000000000000000000000..23674ed0d3cdeb4c5eca2de19178ff295514c351 --- /dev/null +++ b/patches/0006-kv-reuse-prefix.patch @@ -0,0 +1,135 @@ +From: opencoti +Subject: [PATCH 0006] kv-cache reuse for agentic turns — session→slot affinity + +opencoti routes an agentic loop through the bundled llamafile server: each +turn N+1 re-sends turn N's context prefix (system prompt + tool defs + +history) verbatim, then appends. The server already reuses KV for a matching +prefix — cache_prompt defaults to true and get_available_slot() picks the +idle slot whose cached token-prefix best matches the incoming prompt (above +--slot-prompt-similarity, then LRU). That covers single-session turn-2 reuse +at the default --parallel 1. + +What upstream lacks is *session affinity*. With --parallel N and several +concurrent opencode sessions sharing slots, the LCP/LRU picker can route a +session to a slot holding a different session's KV, evicting a prefix that +would otherwise be reused. This patch adds an optional `session_id` request +field that keys slot selection: + + - get_available_slot(): if the request carries a non-empty session_id that + already maps to an idle slot, return that slot (its resident KV is this + session's prefix by construction). The existing LCP→LRU path is the + fallback. + - launch_slot_with_task(): record session_id -> slot.id when a slot is + committed to a session, dropping any stale session previously bound to + that slot. + +The prefix match itself is still the server's existing token-LCP — affinity +only ensures the right slot (the one holding the session's KV) is chosen, so +"(session_id, prefix) reuse" falls out without a separate prefix hash. + +Regression shield: empty session_id (the default, and what every +non-opencoti client sends) skips the affinity phase entirely, leaving slot +selection byte-for-byte identical to upstream. Builds clean under cosmocc +(std::map / std::string already available via server-task.h). + +Bench: perf/llamafile/turn-2-latency.bench.ts +Milestone: F5 M0 — see docs/features/advanced_kv.md +Design: docs/features/advanced_kv.md (M0) +Upstreaming: deferred; candidate to propose upstream as a generic session-id + slot-affinity hint once benched on opencoti workloads. +--- +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -698,6 +698,13 @@ private: + // Necessary similarity of prompt for slot selection + float slot_prompt_similarity = 0.0f; + ++ // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md ++ // session_id -> slot.id affinity, so an agentic session routes back to the ++ // slot holding its resident KV prefix (turn-to-turn reuse, and no ++ // cross-session eviction when --parallel > 1). Empty session_id is never ++ // inserted, so non-opencoti traffic never consults or mutates this map. ++ std::map session_to_slot; ++ + std::string model_name; // name of the loaded model, to be used by API + std::set model_aliases; // additional names for the model + std::set model_tags; // informational tags +@@ -1249,6 +1256,25 @@ private: + + bool update_cache = false; + ++ // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md ++ // Session affinity: if this session already owns an idle slot, reuse ++ // it. That slot's resident KV is the session's prefix, so the LCP path ++ // below would pick it anyway when free — but the explicit pin keeps a ++ // concurrent session from stealing/evicting it under --parallel > 1. ++ // Skipped entirely for empty session_id, so upstream selection (LCP ++ // then LRU, below) is unchanged for every non-opencoti request. ++ if (ret == nullptr && !task.params.session_id.empty()) { ++ auto it = session_to_slot.find(task.params.session_id); ++ if (it != session_to_slot.end()) { ++ for (server_slot & slot : slots) { ++ if (slot.id == it->second && !slot.is_processing()) { ++ ret = &slot; ++ break; ++ } ++ } ++ } ++ } ++ + // find the slot that has at least n% prompt similarity + if (ret == nullptr && slot_prompt_similarity != 0.0f) { + float sim_best = 0; +@@ -1498,6 +1524,22 @@ private: + slot.smpl.reset(); + } + ++ // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md ++ // Record session→slot affinity before `task` is moved into the slot, ++ // so this session's next turn routes back here and reuses its KV. A ++ // slot serves one session at a time, so first drop any stale session ++ // that previously mapped to this slot id. ++ if (!task.params.session_id.empty()) { ++ for (auto it = session_to_slot.begin(); it != session_to_slot.end(); ) { ++ if (it->second == slot.id && it->first != task.params.session_id) { ++ it = session_to_slot.erase(it); ++ } else { ++ ++it; ++ } ++ } ++ session_to_slot[task.params.session_id] = slot.id; ++ } ++ + slot.task = std::make_unique(std::move(task)); + + slot.state = slot.task->is_child() +diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp +--- a/llama.cpp/tools/server/server-task.cpp ++++ b/llama.cpp/tools/server/server-task.cpp +@@ -272,6 +272,7 @@ task_params server_task::params_from_json_cmpl( + params.n_discard = std::max(0, params.n_discard); + params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1)); + params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse); ++ params.session_id = json_value(data, "session_id", std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement + params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); + params.response_fields = json_value(data, "response_fields", std::vector()); +diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h +--- a/llama.cpp/tools/server/server-task.h ++++ b/llama.cpp/tools/server/server-task.h +@@ -61,6 +61,14 @@ struct task_params { + + int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled) + ++ // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md ++ // Optional session affinity key. When non-empty, the server routes this ++ // request back to the slot the same session last used, reusing its ++ // resident KV prefix across agentic turns. Empty (the default, and what ++ // every non-opencoti client sends) preserves upstream slot selection ++ // byte-for-byte. ++ std::string session_id; ++ + // number of prompt tokens before the latest user message + int32_t n_before_user = -1; + diff --git a/patches/0007-build-header-deps.patch b/patches/0007-build-header-deps.patch new file mode 100644 index 0000000000000000000000000000000000000000..990a302763dadc14c27ccc13362a18b8dc6f56ae --- /dev/null +++ b/patches/0007-build-header-deps.patch @@ -0,0 +1,92 @@ +From: opencoti +Subject: [PATCH 0007] build infra — compiler-driven header dependency tracking + +Editing any llama.cpp/src (or common/, tools/server/) header never marked its +dependent objects out of date, so every header change forced a full +from-scratch rebuild — the single biggest drag on iterating the F4/F5 vendored +patches. + +Root cause: the build tracks header deps via cosmopolitan's `mkdeps` +(build/deps.mk → o/$(MODE)/depend). mkdeps only resolves cosmopolitan's +root-relative include style; llama.cpp's sources use `#include "foo.h"` +resolved through `-iquote` dirs, which mkdeps cannot follow, so it emitted +ZERO edges for llama.cpp/src headers. Source->object tracking worked (the +pattern rule), but header->object did not. + +Fix (compiler-driven deps, the modern standard): add `-MMD` to the COMPILE.c / +COMPILE.cc commands so cosmocc emits an accurate per-translation-unit +`.o.d` (project headers only — system headers excluded), then `-include` +those `.o.d` files in build/deps.mk. clang resolves the real `-iquote` +includes, so the header set is exact. cosmocc supports -MMD natively +(get_dependency_outputs in bin/cosmocc): the x86 `.o.d` lands next to the +object and an `.aarch64/*.o.d` twin under the arch subdir; only the x86 copy is +included (header sets are arch-independent). + +Effect: a header edit now recompiles exactly the affected objects (both fat +arches, via the existing rule), and nothing else — no more scratch rebuilds. +The first build after this patch lands populates the .o.d set (a one-time full +compile); every build thereafter is correctly incremental. + +Interaction with ccache: ccache wraps cosmocc's FAT driver, so a cache hit +restores only the x86 .o (never the .aarch64 twin). With accurate header +tracking, objects are no longer manually deleted to force recompiles, so the +.aarch64 twin always persists on disk in a warm tree and hits stay safe. The +remaining hazard — a warm cache restoring x86-only objects into a freshly +reset (o/-wiped) tree — is handled in the opencoti build pipeline, which runs +the full `build` with CCACHE_RECACHE=1. + +Bench: docs/features/llamafile_build.md (incremental-rebuild verification: +touch an included header -> only its dependents recompile; touch a +non-dependency header -> no recompile) +Milestone: build infrastructure (supports F4/F5 — see docs/features/advanced_kv.md) +Upstreaming: build-only; no behavior change to the produced binary. Not +intended for upstream (cosmopolitan/llamafile use mkdeps by design); opencoti +carries it to make iterating the vendored patch series tractable. + +diff --git a/build/deps.mk b/build/deps.mk +--- a/build/deps.mk ++++ b/build/deps.mk +@@ -23,3 +23,15 @@ $(INCS): + rm -f o/$(MODE)/depend + + -include o/$(MODE)/depend ++ ++# opencoti build-header-deps — see docs/features/llamafile_build.md ++# Pull in the per-object dependency files cosmocc emits under -MMD (added to ++# build/rules.mk COMPILE.*). mkdeps' o/$(MODE)/depend above cannot resolve ++# llama.cpp/src's -iquote includes, so it tracked zero of those headers and a ++# header edit never recompiled its dependents — every change forced a ++# from-scratch rebuild. The compiler-emitted `.o.d` files carry the exact ++# header set per translation unit. Only the x86 `.o.d` is pulled in; the ++# `.aarch64/*.o.d` twins list identical headers (arch-independent). The shell ++# find runs at parse time over prior-build outputs (none on a cold build → a ++# harmless empty include, everything compiles anyway). ++-include $(shell [ -d o/$(MODE) ] && find o/$(MODE) -name '*.o.d' -not -path '*/.aarch64/*' 2>/dev/null) +diff --git a/build/rules.mk b/build/rules.mk +--- a/build/rules.mk ++++ b/build/rules.mk +@@ -6,8 +6,22 @@ + # ============================================================================== + + LINK.o = $(CXX) $(CCFLAGS) $(LDFLAGS) +-COMPILE.c = $(CC) $(CCFLAGS) $(CFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -c +-COMPILE.cc = $(CXX) $(CCFLAGS) $(CXXFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -c ++# opencoti build-header-deps — see docs/features/llamafile_build.md ++# -MMD makes cosmocc emit a per-TU dependency file listing the real (project, ++# not system) header set; build/deps.mk -includes them so a header edit ++# recompiles exactly the affected objects. Without this, mkdeps' depend tracked ++# zero llama.cpp/src headers (it can't resolve -iquote includes), so any header ++# change forced a from-scratch rebuild. ++# ++# `-MF $@.d` is REQUIRED, not cosmetic: cosmocc defaults the dep file to ++# `.d` (it appends `.d` → e.g. foo.cpp.o.d), but ccache derives the ++# expected name from `-o foo.cpp.o` by stripping `.o` → foo.cpp.d. That ++# mismatch made ccache stat a file cosmocc never wrote ("failed to stat ++# …foo.cpp.d") and abort every compile with "ccache internal error" → nothing ++# was ever cached. Pinning -MF to cosmocc's actual name (`$@.d` = foo.cpp.o.d) ++# makes the two agree, so ccache caches and restores normally. ++COMPILE.c = $(CC) $(CCFLAGS) $(CFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -MMD -MF $@.d -c ++COMPILE.cc = $(CXX) $(CCFLAGS) $(CXXFLAGS) $(CPPFLAGS_) $(CPPFLAGS) $(TARGET_ARCH) -MMD -MF $@.d -c + + # ============================================================================== + # Standard Compilation Rules diff --git a/patches/0010-rest-kv-eviction.patch b/patches/0010-rest-kv-eviction.patch new file mode 100644 index 0000000000000000000000000000000000000000..503c866a0f047ad1838a9bf9fd907418772efbfd --- /dev/null +++ b/patches/0010-rest-kv-eviction.patch @@ -0,0 +1,476 @@ +From: opencoti +Subject: [PATCH 0010] retention-aware KV eviction — KeyDiff key-similarity, server-window + +When an agentic context exceeds a slot's n_ctx, the server's context-shift +discards the *positional middle* [n_keep, n_keep+n_discard) regardless of what +sits there. This patch makes that choice retention-aware: it discards the +lowest-value *contiguous window* of the same width instead, keeping the +distinctive tokens and dropping the redundant ones. + +Reference choice. The named technique (TRIM-KV, arXiv 2605.09649) needs +per-model retention gates trained by distillation — not a drop-in runtime +patch. Attention-score methods (H2O, SnapKV) need the materialized attention +matrix, which Flash Attention never produces (the vendored pin defaults +flash_attn_type = AUTO). So this substitutes a score-free, FA-compatible +criterion — KeyDiff (arXiv 2504.15364): score each cached key by its +*distinctiveness*, the mean over heads of (1 - cosine(key_head, mean_head)). +Keys close to the sequence's mean are redundant (evict); keys far from it are +distinctive (keep). + +Mechanism (the "server-window" depth — mirror-safe): + - src/llama-memory.h: a new non-pure virtual seq_key_scores(seq_id, + layer_hint) with a default empty body, so no backend is forced to implement + it and the pure-virtual surface is unchanged. (Adding it grows the vtable — + a consistent recompile of all llama.cpp TUs is required; see the build + gotcha below.) + - src/llama-kv-cache.cpp: the real implementation — read each cached key of + one representative layer to host (amortized: only at the rare context-shift + trigger, unlike per-decode-interval scoring), group into heads, return a + per-position distinctiveness vector (+INF for absent positions). Empty when + unsupported (no cells, or a key type with no float converter) so the caller + falls back to positional discard. + - src/llama-kv-cache-iswa.cpp: delegate to the non-SWA base cache. + - src/llama-context.cpp + include/llama.h: a thin C shim + llama_memory_seq_key_scores() — a plain virtual call (no RTTI/downcast). + - tools/server/server-context.cpp: inside the EXISTING context-shift block, + when rest_kv_eviction is on, slide a width-n_discard window over + [n_keep, n_tokens - rest_kv_recent) and discard the minimum-retention one. + It is still ONE contiguous window, so the seq_rm/seq_add + token-mirror + rewrite stay 1:1 with KV positions — no rework of prefix-reuse/sampler. + - common/common.h + common/arg.cpp: --rest-kv-eviction (off by default), + --rest-kv-recent N (protect the recent tail), --rest-kv-layer N (-1 = auto). + +Regression shield: --rest-kv-eviction off (the default, and every non-opencoti +client) leaves w0 = n_keep, i.e. context-shift is byte-for-byte upstream. Any +failure (non-KV memory, quantized keys, no room after the recent tail) also +falls back to the positional middle. + +Build gotcha (buglog bug-175): seq_key_scores is a new virtual on +llama_memory_i, so the vtable layout changes. The llamafile Makefile tracks no +header deps; force a consistent recompile of all llama.cpp src/common/tools +objects (delete x86 .o, rebuild) — and because cosmocc is a fat compiler whose +aarch64 sibling object is NOT restored by a ccache hit, delete only x86 .o that +lack an aarch64 twin and rebuild with CCACHE_DISABLE=1, or use the full +reproducible build:llamafile. + +Bench: perf/llamafile/rest-kv-eviction.bench.ts +Milestone: F5 M1 — see docs/features/advanced_kv.md +Design: docs/features/advanced_kv.md (M1) +Upstreaming: deferred; candidate to propose upstream as a pluggable + context-shift eviction policy once benched on opencoti workloads. +--- +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1398,6 +1398,29 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.ctx_shift = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY}).set_env("LLAMA_ARG_CONTEXT_SHIFT")); ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--rest-kv-eviction"}, ++ {"--no-rest-kv-eviction"}, ++ string_format("when context shift fires, evict the lowest-retention (KeyDiff key-similarity) contiguous window instead of the positional middle (default: %s)", params.rest_kv_eviction ? "enabled" : "disabled"), ++ [](common_params & params, bool value) { ++ params.rest_kv_eviction = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_EVICTION")); ++ add_opt(common_arg( ++ {"--rest-kv-recent"}, "N", ++ string_format("rest-kv: protect the most-recent N tokens from eviction (default: %d)", params.rest_kv_recent), ++ [](common_params & params, int value) { ++ params.rest_kv_recent = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_RECENT")); ++ add_opt(common_arg( ++ {"--rest-kv-layer"}, "N", ++ string_format("rest-kv: model layer whose keys score retention, -1 = auto/middle (default: %d)", params.rest_kv_layer), ++ [](common_params & params, int value) { ++ params.rest_kv_layer = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER")); + add_opt(common_arg( + {"--chunks"}, "N", + string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks), +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -554,6 +554,14 @@ struct common_params { + bool swa_full = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) + bool kv_unified = false; // enable unified KV cache + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // When a slot fills n_ctx and ctx_shift fires, discard the lowest-retention ++ // *contiguous* window (KeyDiff key-similarity) instead of the blind ++ // positional middle. Off => byte-for-byte upstream context-shift. ++ bool rest_kv_eviction = false; // master switch (default off) ++ int32_t rest_kv_recent = 256; // protect the most-recent N tokens from eviction ++ int32_t rest_kv_layer = -1; // representative layer for scoring; -1 = auto (middle) ++ + bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix + bool use_mmap = true; // enable mmap to use filesystem cache + bool use_direct_io = false; // read from disk without buffering +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -730,6 +730,19 @@ extern "C" { + llama_pos p0, + llama_pos p1); + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // Fill out[0..n) with per-position KeyDiff retention scores for seq_id ++ // (higher = keep, +INF = absent position). layer_hint < 0 selects a default ++ // representative layer. Returns the number of finite scores written; 0 when ++ // unsupported (non-KV memory, a key type with no float converter, or an ++ // empty sequence) — the caller should then fall back to positional eviction. ++ LLAMA_API size_t llama_memory_seq_key_scores( ++ llama_memory_t mem, ++ llama_seq_id seq_id, ++ int32_t layer_hint, ++ float * out, ++ size_t n); ++ + // Copy all tokens that belong to the specified sequence to another sequence + // p0 < 0 : [0, p1] + // p1 < 0 : [p0, inf) +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -3732,6 +3732,38 @@ bool llama_memory_seq_rm( + return mem->seq_rm(seq_id, p0, p1); + } + ++// opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++// Thin C shim over the (defaulted) virtual llama_memory_i::seq_key_scores — a ++// plain virtual dispatch, so no RTTI/downcast and no change to the pure-virtual ++// surface of the interface. Non-KV memories inherit the empty default and this ++// returns 0, leaving the server on its positional fallback. ++size_t llama_memory_seq_key_scores( ++ llama_memory_t mem, ++ llama_seq_id seq_id, ++ int32_t layer_hint, ++ float * out, ++ size_t n) { ++ if (!mem || !out || n == 0) { ++ return 0; ++ } ++ ++ const std::vector scores = mem->seq_key_scores(seq_id, layer_hint); ++ if (scores.empty()) { ++ return 0; ++ } ++ ++ size_t cnt = 0; ++ for (size_t p = 0; p < n; ++p) { ++ const float s = p < scores.size() ? scores[p] : std::numeric_limits::infinity(); ++ out[p] = s; ++ if (std::isfinite(s)) { ++ cnt++; ++ } ++ } ++ ++ return cnt; ++} ++ + void llama_memory_seq_cp( + llama_memory_t mem, + llama_seq_id seq_id_src, +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -117,6 +117,14 @@ llama_pos llama_kv_cache_iswa::seq_pos_max(llama_seq_id seq_id) const { + return kv_swa->seq_pos_max(seq_id); + } + ++std::vector llama_kv_cache_iswa::seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const { ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // Score on the non-SWA base cache: the SWA cache only ever retains a ++ // sliding window, so its keys are not the right eviction target. The base ++ // cache is a superset of the SWA cache, so it has every position we score. ++ return kv_base->seq_key_scores(seq_id, layer_hint); ++} ++ + std::map llama_kv_cache_iswa::memory_breakdown() const { + std::map mb = kv_base->memory_breakdown(); + for (const auto & buft_size : kv_swa->memory_breakdown()) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -62,6 +62,9 @@ public: + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ std::vector seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const override; ++ + std::map memory_breakdown() const override; + + // state write/load +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -711,6 +711,137 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + return true; + } + ++std::vector llama_kv_cache::seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const { ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // KeyDiff-style, score-free (Flash-Attention-compatible) retention signal: ++ // for one representative layer, read every cached key of `seq_id`, group ++ // it into attention heads, and score each position by its key's ++ // *distinctiveness* — the mean over heads of (1 - cosine(key_head, ++ // mean_head)). High = far from the sequence's mean key = keep; low = ++ // redundant = safe to evict. The server uses this to choose the ++ // lowest-retention contiguous window to discard during context-shift, ++ // instead of the blind positional middle. Returns scores indexed by token ++ // position, +INF for absent positions (never selected). An empty result ++ // means "unsupported here" (no cells, or keys in a type with no float ++ // converter) and the caller falls back to positional discard. Reads happen ++ // only at the (infrequent) context-shift trigger, so the device->host copy ++ // is amortized — unlike per-decode-interval scoring. ++ ++ if (seq_id < 0 || (size_t) seq_id >= seq_to_stream.size() || layers.empty()) { ++ return {}; ++ } ++ ++ const uint32_t strm = seq_to_stream[seq_id]; ++ const auto & cells = v_cells[strm]; ++ ++ const llama_pos pmax = cells.seq_pos_max(seq_id); ++ if (pmax < 0) { ++ return {}; ++ } ++ ++ // pick a representative cache layer. A non-negative hint first tries to map ++ // a *model* layer id into the cache; failing that it is clamped as a direct ++ // cache-layer index. Default (hint < 0) is the middle layer — empirically ++ // the most informative for retention and cheap (one layer's keys only). ++ int32_t cil = (int32_t) layers.size() / 2; ++ if (layer_hint >= 0) { ++ const auto it = map_layer_ids.find(layer_hint); ++ cil = it != map_layer_ids.end() ++ ? it->second ++ : std::min(layer_hint, (int32_t) layers.size() - 1); ++ } ++ ++ const auto & layer = layers[cil]; ++ const uint32_t mil = layer.il; ++ ++ const uint32_t n_head_kv = hparams.n_head_kv(mil); ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(mil); ++ const uint32_t n_embd_k_gqa = n_head_kv * n_embd_head_k; ++ if (n_head_kv == 0 || n_embd_head_k == 0 || n_embd_k_gqa == 0) { ++ return {}; ++ } ++ ++ ggml_tensor * k = layer.k_stream[strm]; ++ if (!k) { ++ return {}; ++ } ++ ++ const auto * tt = ggml_get_type_traits(k->type); ++ if (!tt || !tt->to_float) { ++ return {}; // e.g. a quantized K cache with no dequantizer -> positional fallback ++ } ++ ++ const size_t row_bytes = ggml_row_size(k->type, n_embd_k_gqa); ++ ++ // collect (pos, cell index) for this sequence, ascending by position ++ std::vector> pc; ++ pc.reserve(cells.get_used()); ++ for (uint32_t i = 0; i < cells.size(); ++i) { ++ if (!cells.is_empty(i) && cells.seq_has(i, seq_id)) { ++ pc.emplace_back(cells.pos_get(i), i); ++ } ++ } ++ if (pc.empty()) { ++ return {}; ++ } ++ std::sort(pc.begin(), pc.end()); ++ ++ const size_t n = pc.size(); ++ ++ // read each key row once (amortized device->host copy), dequantized to f32 ++ std::vector keys((size_t) n * n_embd_k_gqa); ++ std::vector rowbuf(row_bytes); ++ for (size_t j = 0; j < n; ++j) { ++ const uint32_t i = pc[j].second; ++ ggml_backend_tensor_get(k, rowbuf.data(), (size_t) i * row_bytes, row_bytes); ++ tt->to_float(rowbuf.data(), keys.data() + j * n_embd_k_gqa, (int64_t) n_embd_k_gqa); ++ } ++ ++ // per-head mean key across the sequence, then per-head mean norm ++ std::vector mean((size_t) n_embd_k_gqa, 0.0f); ++ for (size_t j = 0; j < n; ++j) { ++ const float * kv = keys.data() + j * n_embd_k_gqa; ++ for (uint32_t e = 0; e < n_embd_k_gqa; ++e) { ++ mean[e] += kv[e]; ++ } ++ } ++ for (uint32_t e = 0; e < n_embd_k_gqa; ++e) { ++ mean[e] /= (float) n; ++ } ++ ++ std::vector mean_norm(n_head_kv, 0.0f); ++ for (uint32_t h = 0; h < n_head_kv; ++h) { ++ const float * mh = mean.data() + (size_t) h * n_embd_head_k; ++ double s = 0.0; ++ for (uint32_t d = 0; d < n_embd_head_k; ++d) { ++ s += (double) mh[d] * (double) mh[d]; ++ } ++ mean_norm[h] = (float) std::sqrt(s); ++ } ++ ++ // distinctiveness per position = mean over heads of (1 - cos(key_head, mean_head)) ++ std::vector out((size_t) pmax + 1, std::numeric_limits::infinity()); ++ for (size_t j = 0; j < n; ++j) { ++ const float * kv = keys.data() + j * n_embd_k_gqa; ++ double acc = 0.0; ++ for (uint32_t h = 0; h < n_head_kv; ++h) { ++ const float * kh = kv + (size_t) h * n_embd_head_k; ++ const float * mh = mean.data() + (size_t) h * n_embd_head_k; ++ double dot = 0.0, kn = 0.0; ++ for (uint32_t d = 0; d < n_embd_head_k; ++d) { ++ dot += (double) kh[d] * (double) mh[d]; ++ kn += (double) kh[d] * (double) kh[d]; ++ } ++ const double denom = std::sqrt(kn) * (double) mean_norm[h]; ++ const double cos = denom > 0.0 ? dot / denom : 0.0; ++ acc += (1.0 - cos); ++ } ++ out[(size_t) pc[j].first] = (float) (acc / (double) n_head_kv); ++ } ++ ++ return out; ++} ++ + void llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + GGML_ASSERT(seq_id_src >= 0 && (size_t) seq_id_src < seq_to_stream.size()); + GGML_ASSERT(seq_id_dst >= 0 && (size_t) seq_id_dst < seq_to_stream.size()); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -146,6 +146,9 @@ public: + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ std::vector seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const override; ++ + std::map memory_breakdown() const override; + + // state write/load +diff --git a/llama.cpp/src/llama-memory.h b/llama.cpp/src/llama-memory.h +--- a/llama.cpp/src/llama-memory.h ++++ b/llama.cpp/src/llama-memory.h +@@ -6,6 +6,7 @@ + #include + #include + #include ++#include + + struct llama_ubatch; + +@@ -112,6 +113,22 @@ struct llama_memory_i { + virtual llama_pos seq_pos_min(llama_seq_id seq_id) const = 0; + virtual llama_pos seq_pos_max(llama_seq_id seq_id) const = 0; + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // Per-position KeyDiff "retention" scores for one sequence: higher = more ++ // distinctive (key far from the sequence's mean key) = keep; lower = more ++ // redundant = safe to evict. The server uses these to pick WHICH window to ++ // discard during context-shift, instead of the blind positional middle. ++ // Returns a vector indexed by token position [0, seq_pos_max]; absent ++ // positions are left at +INF (never evicted). The default returns empty so ++ // non-KV memories — and any caller that never opts in — are byte-for-byte ++ // unaffected (the eviction caller then falls back to positional discard). ++ // layer_hint < 0 selects a default representative layer. ++ virtual std::vector seq_key_scores(llama_seq_id seq_id, int32_t layer_hint) const { ++ (void) seq_id; ++ (void) layer_hint; ++ return {}; ++ } ++ + virtual std::map memory_breakdown() const = 0; + + // +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -2408,14 +2408,63 @@ private: + const int n_left = slot.prompt.n_tokens() - n_keep; + const int n_discard = slot.task->params.n_discard ? slot.task->params.n_discard : (n_left / 2); + ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ // Upstream discards the positional middle [n_keep, n_keep+n_discard). ++ // When rest-kv-eviction is on, slide that same-width window over the ++ // evictable region [n_keep, n_tokens - rest_kv_recent) and discard the ++ // window with the LOWEST retention sum (KeyDiff key-similarity) — keep the ++ // distinctive tokens, drop the redundant ones. It is still ONE contiguous ++ // window, so the seq_rm/seq_add + token-mirror rewrite below stay 1:1 with ++ // KV positions. Any failure (eviction off, non-KV memory, quantized keys, ++ // no room after protecting the recent tail) leaves w0 = n_keep, i.e. the ++ // byte-for-byte upstream positional discard. ++ // opencoti F5 M6 MTP-composition: scoring reads ctx_tgt (the authoritative ++ // full-model KV); the SAME window w0 is then applied to BOTH the target and ++ // the draft KV cache below, preserving target/draft position alignment for ++ // speculative decode. ++ int w0 = n_keep; ++ if (params_base.rest_kv_eviction && n_discard > 0) { ++ const int n_tokens = slot.prompt.n_tokens(); ++ const int recent = std::max(0, params_base.rest_kv_recent); ++ const int hi = n_tokens - recent - n_discard; // last valid window start (inclusive) ++ if (hi >= n_keep) { ++ std::vector scores(n_tokens, std::numeric_limits::infinity()); ++ const size_t got = llama_memory_seq_key_scores( ++ llama_get_memory(ctx_tgt), slot.id, params_base.rest_kv_layer, scores.data(), n_tokens); ++ if (got > 0) { ++ // prefix sums over [n_keep, n_tokens); an absent position gets a ++ // large finite penalty so a window covering one is never picked ++ // (avoids inf/inf arithmetic in the sliding sum). ++ const int m = n_tokens - n_keep; ++ std::vector pref(m + 1, 0.0); ++ for (int i = 0; i < m; ++i) { ++ const float sv = scores[n_keep + i]; ++ pref[i + 1] = pref[i] + (std::isfinite(sv) ? (double) sv : 1e30); ++ } ++ double best = std::numeric_limits::infinity(); ++ int best_w0 = n_keep; ++ for (int s = n_keep; s <= hi; ++s) { ++ const double sum = pref[s - n_keep + n_discard] - pref[s - n_keep]; ++ if (sum < best) { ++ best = sum; ++ best_w0 = s; ++ } ++ } ++ w0 = best_w0; ++ SLT_WRN(slot, "rest-kv eviction: discard [%d, %d) retention=%.3f (vs positional [%d, %d))\n", ++ w0, w0 + n_discard, best, n_keep, n_keep + n_discard); ++ } ++ } ++ } ++ + SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); + +- common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard); +- common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard); ++ common_context_seq_rm (ctx_tgt, slot.id, w0 , w0 + n_discard); ++ common_context_seq_add(ctx_tgt, slot.id, w0 + n_discard, slot.prompt.n_tokens(), -n_discard); + + if (ctx_dft) { +- common_context_seq_rm (ctx_dft.get(), slot.id, n_keep , n_keep + n_discard); +- common_context_seq_add(ctx_dft.get(), slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); ++ common_context_seq_rm (ctx_dft.get(), slot.id, w0 , w0 + n_discard); ++ common_context_seq_add(ctx_dft.get(), slot.id, w0 + n_discard, slot.prompt.tokens.pos_next(), -n_discard); + } + + // add generated tokens to cache +@@ -2424,7 +2473,7 @@ private: + GGML_ASSERT(!slot.prompt.tokens.has_mtmd); + + llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy +- for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) { ++ for (size_t i = w0 + n_discard; i < new_tokens.size(); i++) { + new_tokens[i - n_discard] = new_tokens[i]; + } + diff --git a/patches/0020-headinfer-per-head.patch b/patches/0020-headinfer-per-head.patch new file mode 100644 index 0000000000000000000000000000000000000000..72dee5e90c5a221dd6bb60d93157e9e9b4364969 --- /dev/null +++ b/patches/0020-headinfer-per-head.patch @@ -0,0 +1,520 @@ +From: opencoti +Subject: [PATCH 0020] F5 M2 — HeadInfer: head-wise GPU/CPU KV residency split + +Keep a configurable fraction of KV heads GPU-resident; offload the rest to +host memory; reassemble the full head set via ggml_concat on the head axis +at each attention step. Default off (frac = 1.0). Validated 6/6 on solidPC +RTX 3090 with Qwen2.5-Coder-0.5B-IQ4_XS at -ngl 99: GPU KV halves (48 → 24 +MiB at frac=0.5), CPU half appears (24 MiB), greedy decode byte-identical +between baseline and split. See docs/features/advanced_kv.md (F5 M2). + +The CUDA backend's supports_op for GGML_OP_CONCAT was over-broad (claimed +any non-I32/I16 type) while the kernel ggml_cuda_op_concat only implements +F32. Tightened to match the kernel — F16 concat now correctly routes to the +CPU backend (which has a real F16 path at ggml-cpu/ops.cpp:1980). Required +for the reassembly to run on F16 KV caches. + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1421,6 +1421,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.rest_kv_layer = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER")); ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--headinfer-gpu-heads-frac"}, "F", ++ string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac), ++ [](common_params & params, const std::string & value) { ++ params.headinfer_gpu_heads_frac = std::stof(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC")); + add_opt(common_arg( + {"--chunks"}, "N", + string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks), +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1632,6 +1632,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.slot_initial_ctx = params.slot_initial_ctx < 0 ? 0u : (uint32_t) params.slot_initial_ctx; + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -435,6 +435,11 @@ struct common_params { + // slot_initial_ctx and the over-cap pages are decommitted via + // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior). + int32_t slot_shrink_idle_ms = 0; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Fraction of KV heads kept GPU-resident; the rest are offloaded to host ++ // memory and streamed back per step. 1.0 = off (no split). Only active for ++ // GPU-offloaded layers with flash-attention (non-transposed V cache). ++ float headinfer_gpu_heads_frac = 1.0f; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -5308,8 +5308,16 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15); + case GGML_OP_CONCAT: + { +- ggml_type src0_type = op->src[0]->type; +- return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Tighten to F32-only to match ggml_cuda_op_concat's three ++ // hard asserts at ggml-cuda/concat.cu:158-160. The over-broad ++ // upstream filter ("any type except I32/I16") trusts the ++ // scheduler into dispatching F16 concats to CUDA, where the ++ // kernel asserts; with this fix the scheduler correctly routes ++ // F16 concat to the CPU backend, which has a real F16 path ++ // (ggml-cpu/ops.cpp:1980 concat_f16). Required for M2 get_k / ++ // get_v reassembly on F16 KV caches. ++ return op->src[0]->type == GGML_TYPE_F32; + } break; + case GGML_OP_CONV_TRANSPOSE_1D: + { +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -353,6 +353,11 @@ extern "C" { + // (Phase 2 behavior — soft cap grows monotonically). Requires + // slot_initial_ctx > 0 to have any visible effect. + uint32_t slot_shrink_idle_ms; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Fraction of KV heads kept GPU-resident (rest offloaded to host and ++ // streamed back per step). 1.0 = off. Only effective for offloaded ++ // layers with flash-attention (non-transposed V cache). ++ float headinfer_gpu_heads_frac; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -62,6 +62,8 @@ llama_context::llama_context( + cparams.slot_initial_ctx = params.slot_initial_ctx; + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -3353,6 +3355,7 @@ llama_context_params llama_context_default_params() { + /*.n_seq_max =*/ 1, + /*.slot_initial_ctx =*/ 0, + /*.slot_shrink_idle_ms =*/ 0, ++ /*.headinfer_gpu_heads_frac =*/ 1.0f, + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -21,6 +21,8 @@ struct llama_cparams { + // milliseconds: when a grown KV-cache is idle for at least this long, the + // soft cap is reset to slot_initial_ctx and the pages beyond are decommitted. + uint32_t slot_shrink_idle_ms; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ float headinfer_gpu_heads_frac; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -22,6 +22,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + uint32_t kv_size, + uint32_t kv_size_initial, + uint32_t slot_shrink_idle_ms, ++ float headinfer_gpu_heads_frac, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +@@ -64,6 +65,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + kv_base = std::make_unique( + model, type_k, type_v, + v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, ++ headinfer_gpu_heads_frac, + n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); +@@ -71,6 +73,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + kv_swa = std::make_unique( + model, type_k, type_v, + v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, ++ headinfer_gpu_heads_frac, + n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse); + } + +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -28,6 +28,8 @@ public: + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + // Forwarded to both kv_base and kv_swa. 0 = shrink disabled. + uint32_t slot_shrink_idle_ms, ++ // opencoti F5 M2 headinfer — forwarded to both kv_base and kv_swa. ++ float headinfer_gpu_heads_frac, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -110,6 +110,7 @@ llama_kv_cache::llama_kv_cache( + uint32_t kv_size, + uint32_t kv_size_initial, + uint32_t slot_shrink_idle_ms, ++ float headinfer_gpu_heads_frac, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -262,23 +263,67 @@ llama_kv_cache::llama_kv_cache( + const bool has_k = true; + const bool has_v = !is_mla; + +- ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gqa, kv_size, n_stream) : nullptr; +- ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gqa, kv_size, n_stream) : nullptr; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Head-residency split: keep the first gpu_heads KV heads on this ++ // (offloaded/device) layer's buffer and put the remaining heads on a ++ // host buffer. Splitting on KV-head boundaries is inherently GQA-safe. ++ // Gated to offloaded, non-transposed-V, unified (n_stream==1) layers; ++ // frac >= 1.0 (or <= 1 head) means no split (byte-identical upstream). ++ uint32_t gpu_heads = 0; ++ uint32_t n_embd_k_gpu = n_embd_k_gqa; ++ uint32_t n_embd_v_gpu = n_embd_v_gqa; ++ uint32_t n_embd_k_cpu = 0; ++ uint32_t n_embd_v_cpu = 0; ++ if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans && n_stream == 1) { ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ if (n_head_kv > 1) { ++ uint32_t g = (uint32_t) std::lround((double) headinfer_gpu_heads_frac * (double) n_head_kv); ++ g = std::max(1u, std::min(g, n_head_kv - 1)); ++ gpu_heads = g; ++ n_embd_k_gpu = g * n_embd_head_k; ++ n_embd_v_gpu = g * n_embd_head_v; ++ n_embd_k_cpu = (n_head_kv - g) * n_embd_head_k; ++ n_embd_v_cpu = (n_head_kv - g) * n_embd_head_v; ++ LLAMA_LOG_INFO("%s: layer %3d: headinfer split heads %u GPU / %u CPU\n", ++ __func__, il, gpu_heads, n_head_kv - gpu_heads); ++ } ++ } ++ ++ ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gpu, kv_size, n_stream) : nullptr; ++ ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gpu, kv_size, n_stream) : nullptr; + + has_k && ggml_format_name(k, "cache_k_l%d", il); + has_v && ggml_format_name(v, "cache_v_l%d", il); + ++ // opencoti F5 M2 headinfer — the host-resident head subset lives in the ++ // CPU buffer-type context; the buffer-allocation loop below picks it up ++ // automatically (ctx_for_buft keys contexts by buffer type). ++ ggml_tensor * k_cpu = nullptr; ++ ggml_tensor * v_cpu = nullptr; ++ if (gpu_heads > 0) { ++ ggml_context * ctx_cpu = ctx_for_buft(ggml_backend_cpu_buffer_type()); ++ if (!ctx_cpu) { ++ throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); ++ } ++ k_cpu = has_k ? ggml_new_tensor_3d(ctx_cpu, type_k, n_embd_k_cpu, kv_size, n_stream) : nullptr; ++ v_cpu = has_v ? ggml_new_tensor_3d(ctx_cpu, type_v, n_embd_v_cpu, kv_size, n_stream) : nullptr; ++ has_k && k_cpu && ggml_format_name(k_cpu, "cache_k_cpu_l%d", il); ++ has_v && v_cpu && ggml_format_name(v_cpu, "cache_v_cpu_l%d", il); ++ } ++ + std::vector k_stream; + std::vector v_stream; + + for (uint32_t s = 0; s < n_stream; ++s) { +- k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gqa, kv_size, k->nb[1], s*k->nb[2]) : nullptr); +- v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr); ++ k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gpu, kv_size, k->nb[1], s*k->nb[2]) : nullptr); ++ v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gpu, kv_size, v->nb[1], s*v->nb[2]) : nullptr); + } + + map_layer_ids[il] = layers.size(); + +- layers.push_back({ il, k, v, k_stream, v_stream, }); ++ layers.push_back({ il, k, v, k_stream, v_stream, k_cpu, v_cpu, gpu_heads }); + } + + if (reuse) { +@@ -440,6 +485,20 @@ void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) { + const size_t sz = (size_t) (b - a) * (size_t) layer.v->nb[1]; + ggml_backend_tensor_memset(layer.v, 0, off, sz); + } ++ // opencoti F5 M2 headinfer — clear the host-resident head subset ++ // too (its own strides; same [a, b) cell range). ++ if (layer.k_cpu) { ++ const size_t off = (size_t) st * (size_t) layer.k_cpu->nb[2] ++ + (size_t) a * (size_t) layer.k_cpu->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) layer.k_cpu->nb[1]; ++ ggml_backend_tensor_memset(layer.k_cpu, 0, off, sz); ++ } ++ if (layer.v_cpu) { ++ const size_t off = (size_t) st * (size_t) layer.v_cpu->nb[2] ++ + (size_t) a * (size_t) layer.v_cpu->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) layer.v_cpu->nb[1]; ++ ggml_backend_tensor_memset(layer.v_cpu, 0, off, sz); ++ } + } + } + +@@ -610,6 +669,11 @@ void llama_kv_cache::shrink_if_idle() { + for (const auto & layer : layers) { + opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells); + opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells); ++ // opencoti F5 M2 headinfer — the host-resident head subset IS ++ // host-pageable, so this is where the GPU-mode shrink actually returns ++ // RSS (the device k/v above are skipped by the is_host() guard). ++ opencoti_decommit_layer_range(layer.k_cpu, n_stream, from_cells, to_cells); ++ opencoti_decommit_layer_range(layer.v_cpu, n_stream, from_cells, to_cells); + } + + // After decommit, treat cells [from_cells, to_cells) as "uncleared" — +@@ -1645,15 +1709,41 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { + ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + +- auto * k = layers[ikv].k; ++ const auto & layer = layers[ikv]; ++ auto * k = layer.k; ++ ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ ++ // opencoti F5 M2 headinfer — reassemble the full head set from the ++ // GPU-resident (k) and host-resident (k_cpu) subsets by concat on the head ++ // dimension. The backend scheduler streams k_cpu to the compute backend. ++ if (layer.gpu_heads > 0 && layer.k_cpu) { ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_gpu = layer.gpu_heads; ++ const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; ++ const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_k; ++ const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_k; ++ auto * kc = layer.k_cpu; ++ ggml_tensor * kg_v = ggml_view_4d(ctx, k, ++ n_embd_head_k, n_head_gpu, n_kv, ns, ++ ggml_row_size(k->type, n_embd_head_k), ++ ggml_row_size(k->type, n_embd_gpu), ++ ggml_row_size(k->type, n_embd_gpu*kv_size), ++ ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0); ++ ggml_tensor * kc_v = ggml_view_4d(ctx, kc, ++ n_embd_head_k, n_head_cpu, n_kv, ns, ++ ggml_row_size(kc->type, n_embd_head_k), ++ ggml_row_size(kc->type, n_embd_cpu), ++ ggml_row_size(kc->type, n_embd_cpu*kv_size), ++ ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0); ++ return ggml_concat(ctx, kg_v, kc_v, 1); ++ } + +- const uint64_t kv_size = get_size(); + const uint64_t n_embd_k_gqa = k->ne[0]; + + assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il)); + +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; +- + return ggml_view_4d(ctx, k, + hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns, + ggml_row_size(k->type, hparams.n_embd_head_k(il)), +@@ -1665,16 +1755,42 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k + ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + +- auto * v = layers[ikv].v; ++ const auto & layer = layers[ikv]; ++ auto * v = layer.v; ++ ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ ++ // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The split ++ // is gated off when v_trans at construction, so this branch is always the ++ // non-transposed layout (heads in dim 1, same as get_k). ++ if (layer.gpu_heads > 0 && layer.v_cpu) { ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_gpu = layer.gpu_heads; ++ const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; ++ const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_v; ++ const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_v; ++ auto * vc = layer.v_cpu; ++ ggml_tensor * vg_v = ggml_view_4d(ctx, v, ++ n_embd_head_v, n_head_gpu, n_kv, ns, ++ ggml_row_size(v->type, n_embd_head_v), ++ ggml_row_size(v->type, n_embd_gpu), ++ ggml_row_size(v->type, n_embd_gpu*kv_size), ++ ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0); ++ ggml_tensor * vc_v = ggml_view_4d(ctx, vc, ++ n_embd_head_v, n_head_cpu, n_kv, ns, ++ ggml_row_size(vc->type, n_embd_head_v), ++ ggml_row_size(vc->type, n_embd_cpu), ++ ggml_row_size(vc->type, n_embd_cpu*kv_size), ++ ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0); ++ return ggml_concat(ctx, vg_v, vc_v, 1); ++ } + +- const uint64_t kv_size = get_size(); + const uint64_t n_embd_v_gqa = v->ne[0]; + + // [TAG_V_CACHE_VARIABLE] + assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il)); + +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; +- + if (!v_trans) { + // note: v->nb[1] <= v->nb[2] + return ggml_view_4d(ctx, v, +@@ -1699,7 +1815,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + + const int32_t ikv = map_layer_ids.at(il); + +- ggml_tensor * k = layers[ikv].k; ++ const auto & layer = layers[ikv]; ++ ggml_tensor * k = layer.k; + + const int64_t n_embd_head = k_cur->ne[0]; + const int64_t n_head = k_cur->ne[1]; +@@ -1711,6 +1828,24 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + // TODO: add ggml helper function for this? + GGML_ASSERT(ggml_row_size(k_cur->type, n_embd_head) == k_cur->nb[1]); + ++ // opencoti F5 M2 headinfer — scatter the incoming heads into the GPU and ++ // host subsets. The first gpu_heads heads (contiguous in k_cur's dim 0..1) ++ // store into k; the rest store into k_cpu, both at the same cell indices ++ // (k_idxs). The single returned dep node ties both set_rows into the ++ // caller's one ggml_build_forward_expand (its value is never read). Split ++ // only happens for n_stream==1 (gated at construction) — no per-stream ++ // reshape needed. ++ if (layer.gpu_heads > 0 && layer.k_cpu) { ++ const int64_t g = layer.gpu_heads; ++ const int64_t c = n_head - g; ++ ggml_tensor * kc = layer.k_cpu; ++ ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur, n_embd_head*g, n_tokens, k_cur->nb[2], 0); ++ ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur, n_embd_head*c, n_tokens, k_cur->nb[2], (size_t) (g*k_cur->nb[1])); ++ ggml_tensor * store_g = ggml_set_rows(ctx, k, cur_g, k_idxs); ++ ggml_tensor * store_c = ggml_set_rows(ctx, kc, cur_c, k_idxs); ++ return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0)); ++ } ++ + k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0); + + const int64_t n_stream = k->ne[2]; +@@ -1734,7 +1869,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + + const int32_t ikv = map_layer_ids.at(il); + +- auto * v = layers[ikv].v; ++ const auto & layer = layers[ikv]; ++ auto * v = layer.v; + + const int64_t n_embd_head = v_cur->ne[0]; + const int64_t n_head = v_cur->ne[1]; +@@ -1745,6 +1881,20 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + // we can merge dims 0 and 1 + GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]); + ++ // opencoti F5 M2 headinfer — head-scatter into the GPU + host subsets. ++ // Reached only when !v_trans and n_stream==1 (the split is gated off ++ // otherwise at construction), so this mirrors the cpy_k store exactly. ++ if (layer.gpu_heads > 0 && layer.v_cpu) { ++ const int64_t g = layer.gpu_heads; ++ const int64_t c = n_head - g; ++ ggml_tensor * vc = layer.v_cpu; ++ ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur, n_embd_head*g, n_tokens, v_cur->nb[2], 0); ++ ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur, n_embd_head*c, n_tokens, v_cur->nb[2], (size_t) (g*v_cur->nb[1])); ++ ggml_tensor * store_g = ggml_set_rows(ctx, v, cur_g, v_idxs); ++ ggml_tensor * store_c = ggml_set_rows(ctx, vc, cur_c, v_idxs); ++ return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0)); ++ } ++ + const int64_t n_stream = v->ne[2]; + + // take this branch when FA is enabled (the V cache is not transposed) +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -111,6 +111,11 @@ public: + // back to kv_size_initial and the over-cap pages are + // decommitted. 0 disables shrink-on-idle. + uint32_t slot_shrink_idle_ms, ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Fraction of KV heads kept GPU-resident (rest offloaded ++ // to host, streamed back per step). 1.0 = off. Effective ++ // only for offloaded layers with !v_trans (flash-attn). ++ float headinfer_gpu_heads_frac, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -269,6 +274,14 @@ private: + + std::vector k_stream; + std::vector v_stream; ++ ++ // opencoti F5 M2 headinfer — head-residency split. When gpu_heads > 0, ++ // `k`/`v` hold the first gpu_heads KV heads on the layer's (GPU) buffer ++ // and `k_cpu`/`v_cpu` hold the remaining (n_head_kv - gpu_heads) heads ++ // on a host buffer. gpu_heads == 0 means no split (k/v hold all heads). ++ ggml_tensor * k_cpu = nullptr; ++ ggml_tensor * v_cpu = nullptr; ++ uint32_t gpu_heads = 0; + }; + + bool v_trans = true; // the value tensor is transposed +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -48,6 +48,8 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + // Hybrid-iswa does not yet thread slot_shrink_idle_ms; pass 0 for + // Phase 2 backward-compat (no shrink-on-idle). + 0, ++ // opencoti F5 M2 headinfer — hybrid-iswa does not thread headinfer; 1.0 = off. ++ 1.0f, + n_seq_max, + n_ubatch, + n_pad, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -47,6 +47,8 @@ llama_memory_hybrid::llama_memory_hybrid( + // Hybrid memory does not yet thread slot_shrink_idle_ms; pass 0 for + // Phase 2 backward-compat (no shrink-on-idle). + 0, ++ // opencoti F5 M2 headinfer — hybrid memory does not thread headinfer; 1.0 = off. ++ 1.0f, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2077,6 +2077,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.slot_initial_ctx, + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_shrink_idle_ms, ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ cparams.headinfer_gpu_heads_frac, + cparams.n_seq_max, + cparams.n_ubatch, + 1, +@@ -2097,6 +2099,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.slot_initial_ctx, + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + cparams.slot_shrink_idle_ms, ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ cparams.headinfer_gpu_heads_frac, + cparams.n_seq_max, + 1, + hparams.n_swa, diff --git a/patches/0030-neo-pipeline.patch b/patches/0030-neo-pipeline.patch new file mode 100644 index 0000000000000000000000000000000000000000..c8f42f0a2753dff6f53c55200be14fdd492a6bb9 --- /dev/null +++ b/patches/0030-neo-pipeline.patch @@ -0,0 +1,933 @@ +From: opencoti +Subject: [PATCH 0030] F5 M3 — NEO: asymmetric GPU/CPU attention pipelining + +Sits on top of 0020 (M2 head-residency split). When --neo-pipeline != off and +the headinfer split is active for the layer, build_attn_mha builds TWO +ggml_flash_attn_ext ops on disjoint Q-head ranges instead of M2's +concat-then-stream-back path: cur_g consumes the GPU half of K/V on the GPU +backend; cur_c consumes the CPU half of K/V on the CPU backend (auto-routed +by the scheduler from each FA op's K/V buffer-type). Output halves are +ggml_concat'd on the head axis — same head-axis concat M2 used, but on +O(n_tokens) FA outputs instead of O(n_kv) K/V. + +A small orchestrator (ggml/include/ggml-neo-pipeline.h + ggml/src/ +ggml-neo-pipeline.cpp) lets the graph builder register each (cur_g, cur_c) +pair; the CUDA backend's graph_compute hook routes registered cur_g to a +non-default CUDA stream (curr_stream_no=1) so concurrent CPU FA on cur_c can +overlap with stream-1 GPU FA. A cudaEvent is recorded on stream 1 after the +GPU FA dispatch; when a downstream split's graph reads cur_g (typically the +concat join), the hook inserts a cudaStreamWaitEvent on the consuming stream +so the read sees completed data. CUDA graph capture is disabled for +NEO-engaged splits; off-path graphs keep the fast cuda_graph path. + +Default off in every layer: cparams.neo_pipeline_mode == 0 means +build_attn_mha_neo is never called and the M3-A structural split never +engages — the M2 concat-then-stream-back path runs unchanged, byte-identical +to the 0020-shipped binary. Adapter exposes --neo-pipeline off|on|auto via +common/arg.cpp; @opencoti/llamafile typed config field +neoPipelineMode + OPENCOTI_LLAMAFILE_NEO_PIPELINE env mirror the M2 +template. + +Solidpc RTX 3090 + Qwen2.5-Coder-0.5B IQ4_XS bench (perf/llamafile/ +neo-pipeline.bench.ts) at ctx=4096 / predict=128 / frac=0.5: + + B baseline (no split) 326.69 tps + M m2-only (split, M2 path) 186.96 tps + N neo-on (split, M3 path) 187.04 tps (+0.04% vs M) ← C2 FAIL + O off-id (--neo-pipeline off) 151.65 tps + + C1 PASS N completion === B completion (correctness intact) + C2 FAIL N tps - M tps < 5% (NEO overlap mechanism is correct but the + per-step GPU FA on half-heads finishes in much less wall time + than the CPU FA on half-heads, so even perfect overlap is + bounded by the slower CPU half — for this model + ctx, + M3-B's stream-swap has no observable win over M3-A's + structural split alone) + C3 PASS N GPU KV (24 MiB) ≈ M GPU KV (24 MiB) — residency preserved + C4 PASS O completion === M completion (--neo-pipeline off byte-identical) + +Per the M3-D gate in docs/features/advanced_kv.md (F5 M3): C2 failed but +the orchestrator is correct and the off-path is byte-identical to M2, +so the orchestrator file ships as a wired-but-no-observable-win +mechanism (kept for M4 ScoutAttention reuse) rather than as a +dormant stub. The M3-A structural win (per-step CPU→GPU stream-back +eliminated) is the actual deliverable. M3-C EWMA load-aware controller +is deferred — without an observable M3-B overlap win it would be +cosmetic. + +Surgical-hook count stays 18 — all 0030 changes are in vendored-source +files, captured here via snapshot-diff (NOT git diff HEAD; bug-121). + +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -27,6 +27,7 @@ GGML_SRCS_CPP := \ + llama.cpp/ggml/src/ggml-backend-meta.cpp \ + llama.cpp/ggml/src/ggml-backend-reg.cpp \ + llama.cpp/ggml/src/ggml-backend.cpp \ ++ llama.cpp/ggml/src/ggml-neo-pipeline.cpp \ + llama.cpp/ggml/src/ggml-opt.cpp \ + llama.cpp/ggml/src/ggml-threading.cpp \ + llama.cpp/ggml/src/ggml.cpp \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1429,6 +1429,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.headinfer_gpu_heads_frac = std::stof(value); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC")); ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--neo-pipeline"}, "MODE", ++ "neo (asymmetric GPU/CPU attention pipelining): off (default), on (engage when headinfer split is active for the layer), auto (reserved, same as on). Requires --headinfer-gpu-heads-frac < 1.0 to do anything.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off") params.neo_pipeline_mode = 0; ++ else if (value == "on") params.neo_pipeline_mode = 1; ++ else if (value == "auto") params.neo_pipeline_mode = 2; ++ else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); + add_opt(common_arg( + {"--chunks"}, "N", + string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks), +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1634,6 +1634,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ cparams.neo_pipeline_mode = params.neo_pipeline_mode; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -440,6 +440,11 @@ struct common_params { + // memory and streamed back per step. 1.0 = off (no split). Only active for + // GPU-offloaded layers with flash-attention (non-transposed V cache). + float headinfer_gpu_heads_frac = 1.0f; ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ // 0 = off (default; M2 concat path runs), 1 = on (engage when split ++ // active for the layer), 2 = auto (reserved for future load-based ++ // engagement; same as `on` today). ++ uint32_t neo_pipeline_mode = 0; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/ggml/include/ggml-neo-pipeline.h b/llama.cpp/ggml/include/ggml-neo-pipeline.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/include/ggml-neo-pipeline.h +@@ -0,0 +1,78 @@ ++// opencoti F5 M3 NEO — asymmetric GPU/CPU attention pipelining. ++// ++// This header declares the orchestrator surface that lets the M3 two-flash-attn ++// structural split overlap its GPU and CPU halves. The graph builder registers ++// each (cur_g, cur_c) pair after constructing it; the CUDA backend consults the ++// registry at compute time to (a) dispatch GPU FA on a non-default CUDA stream ++// so the scheduler's stream-0 sync no longer blocks the CPU FA's input copy ++// and (b) wait on the recorded stream-1 event when a downstream split consumes ++// cur_g (notably the concat that joins both halves), keeping correctness intact. ++// ++// Default-off: ggml_neo_pipeline_set_enabled(false) makes register_pair a no-op ++// and the CUDA hooks fast-skip. The off-path is byte-identical to the M3-A ++// shipped path; M3-A in turn is byte-identical to M2's concat path when ++// --neo-pipeline off (cparams.neo_pipeline_mode == 0). See ++// docs/features/advanced_kv.md (F5 M3 NEO). ++#pragma once ++ ++#include "ggml.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++// Global on/off switch. Off by default; the graph builder flips this on for ++// the duration of a graph build when cparams.neo_pipeline_mode != 0 and at ++// least one layer has an active headinfer split. While off, all functions in ++// this file are O(1) no-ops. ++GGML_API void ggml_neo_pipeline_set_enabled(bool enabled); ++GGML_API bool ggml_neo_pipeline_enabled(void); ++ ++// Registers a NEO FA pair. Called from build_attn_mha_neo after constructing ++// the two flash-attention ops on disjoint head ranges. cur_g lives on the GPU ++// backend (CUDA-allocated K/V), cur_c on the CPU backend (CPU-allocated K/V). ++// The CUDA backend later consults this registry to route cur_g to a non-default ++// stream so cur_c can dispatch concurrently. When disabled, this is a no-op. ++GGML_API void ggml_neo_pipeline_register_pair(const struct ggml_tensor * cur_g, ++ const struct ggml_tensor * cur_c); ++ ++// Membership probe. Returns true iff t was the cur_g argument of a prior ++// register_pair call in this graph. O(1) fast-path; the CUDA backend calls ++// this once per node when scanning a graph it is about to dispatch. ++GGML_API bool ggml_neo_pipeline_is_gpu_partner(const struct ggml_tensor * t); ++ ++// CUDA event slot for cur_g. The CUDA backend creates a cudaEvent (with ++// cudaEventCreateWithFlags + cudaEventDisableTiming) the first time it ++// dispatches cur_g on a non-default stream and stores the opaque handle via ++// set_event. A later split whose graph reads cur_g calls get_event and ++// inserts a cudaStreamWaitEvent so the consuming stream waits for the ++// stream-1 work. Both functions are no-ops when NEO is disabled or t isn't ++// registered (get_event returns NULL). The void* is treated as cudaEvent_t ++// at the CUDA call site; the orchestrator does not know the underlying type. ++GGML_API void * ggml_neo_pipeline_get_event(const struct ggml_tensor * cur_g); ++GGML_API void ggml_neo_pipeline_set_event(const struct ggml_tensor * cur_g, void * event); ++ ++// Clears all per-graph registrations. Called by the CUDA backend at the end ++// of each graph dispatch so registrations don't leak across decode steps. ++// (The graph builder reinstalls them on the next build.) Notably does NOT ++// destroy stored events — those are destroyed by the CUDA backend, which ++// owns the cudaEvent lifecycle (call destroy_all_events at shutdown). ++GGML_API void ggml_neo_pipeline_clear_registrations(void); ++ ++// Walks every (cur_g, event) entry, invoking destroyer(event) for each ++// non-null event, then clears the event map. Called at session/backend ++// shutdown so cudaEvents are properly released. destroyer is typically a ++// thin lambda wrapping cudaEventDestroy at the CUDA call site. ++GGML_API void ggml_neo_pipeline_destroy_all_events(void (*destroyer)(void * event)); ++ ++// Counter of NEO pair dispatches observed by the CUDA backend in the current ++// process. Surfaced for the M3-E bench's residency / engagement checks. ++GGML_API int ggml_neo_pipeline_dispatch_count(void); ++ ++// Increments the dispatch counter. Called by the CUDA backend each time it ++// observes and routes a registered cur_g to a non-default stream. ++GGML_API void ggml_neo_pipeline_inc_dispatch_count(void); ++ ++#ifdef __cplusplus ++} ++#endif +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -2,6 +2,10 @@ + #include "ggml-impl.h" + #include "ggml-backend-impl.h" + ++// opencoti F5 M3-B NEO orchestrator (registry of cur_g/cur_c pairs for the ++// asymmetric attention pipelining; consulted in graph_compute below). ++#include "ggml-neo-pipeline.h" ++ + #include "ggml-cuda/allreduce.cuh" + #include "ggml-cuda/common.cuh" + #include "ggml-cuda/acc.cuh" +@@ -4485,6 +4489,58 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t + + ggml_cuda_set_device(cuda_ctx->device); + ++ // opencoti F5 M3-B NEO: scan the graph for registered cur_g flash-attn ++ // outputs and for downstream consumers of prior cur_g. The orchestrator ++ // is dormant unless the graph builder called set_enabled(true) (which ++ // only happens when --neo-pipeline != off AND at least one layer has ++ // an active headinfer split). When dormant this loop body is one ++ // atomic-load fast-skip. ++ // ++ // When NEO is engaged for this split: ++ // - If the split computes a cur_g, swap curr_stream_no to 1 so the ++ // kernels dispatch on the alternate CUDA stream; the scheduler's ++ // stream-0 sync in compute_splits will NOT block on this work, so ++ // the subsequent split's q_c host copy starts immediately and the ++ // CPU FA on cur_c can overlap with our stream-1 work. We record an ++ // event on stream 1 at exit so downstream consumers know when ++ // cur_g is done. ++ // - If the split reads a prior cur_g (typically the concat join), ++ // stream-wait on the recorded event before kernels read it. ++ // ++ // CUDA graph capture is disabled when NEO is active for this split: ++ // capturing kernel dispatch on a non-default stream interacts with the ++ // graph_key cache and the existing concurrent_event re-ordering pass ++ // in ways we haven't validated. Off-path graphs keep the cuda_graph ++ // fast path; only NEO-engaged splits pay this cost. ++ bool neo_active = false; ++ ggml_tensor * neo_cur_g_to_record = nullptr; ++ if (ggml_neo_pipeline_enabled()) { ++ for (int i = 0; i < cgraph->n_nodes; ++i) { ++ ggml_tensor * node = cgraph->nodes[i]; ++ if (ggml_neo_pipeline_is_gpu_partner(node)) { ++ neo_active = true; ++ neo_cur_g_to_record = node; ++ break; ++ } ++ for (int s = 0; s < GGML_MAX_SRC; ++s) { ++ if (node->src[s] != nullptr && ggml_neo_pipeline_is_gpu_partner(node->src[s])) { ++ void * ev = ggml_neo_pipeline_get_event(node->src[s]); ++ if (ev != nullptr) { ++ CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t) ev, 0)); ++ } ++ } ++ } ++ } ++ } ++ if (neo_active) { ++ // Materialize stream 1 (creates it on first use) and switch the ++ // context to dispatch all subsequent kernels there. Both calls are ++ // cheap: stream() is a vector lookup + lazy create; the curr_stream_no ++ // write is a single int. ++ (void) cuda_ctx->stream(cuda_ctx->device, 1); ++ cuda_ctx->curr_stream_no = 1; ++ } ++ + bool use_cuda_graph = false; + bool cuda_graph_update_required = false; + const void * graph_key = nullptr; +@@ -4524,6 +4580,16 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t + } + #endif // USE_CUDA_GRAPH + ++ // opencoti F5 M3-B NEO: force-disable cuda_graph capture for NEO-active ++ // splits. Capturing on a non-default stream is structurally fine in CUDA ++ // but interacts with the existing graph_key cache + concurrent_event ++ // re-ordering pass in ways we haven't validated. Off-path graphs keep ++ // the fast cuda_graph path. ++ if (neo_active) { ++ use_cuda_graph = false; ++ cuda_graph_update_required = false; ++ } ++ + if (use_cuda_graph && cuda_graph_update_required) { + // Start CUDA graph capture + { +@@ -4536,6 +4602,23 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t + + ggml_cuda_graph_evaluate_and_capture(cuda_ctx, cgraph, use_cuda_graph, cuda_graph_update_required, graph_key); + ++ // opencoti F5 M3-B NEO exit: record an event on stream 1 marking cur_g's ++ // completion so the downstream concat (consumer split) can stream-wait ++ // on it from stream 0. Restore curr_stream_no=0 so the next graph_compute ++ // call starts on the default stream as normal. ++ if (neo_active && neo_cur_g_to_record != nullptr) { ++ void * ev = ggml_neo_pipeline_get_event(neo_cur_g_to_record); ++ if (ev == nullptr) { ++ cudaEvent_t new_ev; ++ CUDA_CHECK(cudaEventCreateWithFlags(&new_ev, cudaEventDisableTiming)); ++ ev = (void *) new_ev; ++ ggml_neo_pipeline_set_event(neo_cur_g_to_record, ev); ++ } ++ CUDA_CHECK(cudaEventRecord((cudaEvent_t) ev, cuda_ctx->stream(cuda_ctx->device, 1))); ++ ggml_neo_pipeline_inc_dispatch_count(); ++ cuda_ctx->curr_stream_no = 0; ++ } ++ + return GGML_STATUS_SUCCESS; + } + +diff --git a/llama.cpp/ggml/src/ggml-neo-pipeline.cpp b/llama.cpp/ggml/src/ggml-neo-pipeline.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-neo-pipeline.cpp +@@ -0,0 +1,126 @@ ++// opencoti F5 M3 NEO orchestrator implementation. ++// ++// Backend-agnostic registry that lets the CUDA backend recognize the M3-built ++// (cur_g, cur_c) flash-attention pairs and route cur_g to a non-default CUDA ++// stream so cur_c can dispatch on the CPU backend concurrently. ++// ++// The orchestrator stores tensor pointers and opaque cudaEvent handles. It ++// does not call any CUDA API directly — the CUDA backend creates events with ++// cudaEventCreateWithFlags and hands them to set_event; on shutdown the CUDA ++// backend supplies a destroyer callback that wraps cudaEventDestroy. ++// ++// All maps are protected by a single mutex. Lookups are O(1) (unordered_map). ++// While the global enabled flag is false, every public function fast-skips ++// without touching the maps, so the off-path is one atomic-load + branch. ++// ++// See ggml/include/ggml-neo-pipeline.h for the public API contract and ++// docs/features/advanced_kv.md (F5 M3) for the surrounding design. ++ ++#include "ggml-neo-pipeline.h" ++ ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++std::atomic g_enabled{false}; ++std::atomic g_dispatch_count{0}; ++ ++std::mutex g_mu; ++ ++// Registered cur_g tensors. Membership probe is O(1). ++std::unordered_set g_gpu_partners; ++ ++// Map from cur_g → cudaEvent_t handle (opaque void *). NULL slot means the ++// CUDA backend hasn't created an event for this tensor yet — get_event ++// returns nullptr in that case so the caller knows to create-and-set. ++std::unordered_map g_events; ++ ++} // namespace ++ ++void ggml_neo_pipeline_set_enabled(bool enabled) { ++ g_enabled.store(enabled, std::memory_order_release); ++} ++ ++bool ggml_neo_pipeline_enabled(void) { ++ return g_enabled.load(std::memory_order_acquire); ++} ++ ++void ggml_neo_pipeline_register_pair(const ggml_tensor * cur_g, const ggml_tensor * cur_c) { ++ (void) cur_c; // cpu partner is informational; the orchestrator only acts on cur_g ++ if (!ggml_neo_pipeline_enabled()) { ++ return; ++ } ++ if (cur_g == nullptr) { ++ return; ++ } ++ std::lock_guard lock(g_mu); ++ g_gpu_partners.insert(cur_g); ++} ++ ++bool ggml_neo_pipeline_is_gpu_partner(const ggml_tensor * t) { ++ if (!ggml_neo_pipeline_enabled()) { ++ return false; ++ } ++ if (t == nullptr) { ++ return false; ++ } ++ std::lock_guard lock(g_mu); ++ return g_gpu_partners.find(t) != g_gpu_partners.end(); ++} ++ ++void * ggml_neo_pipeline_get_event(const ggml_tensor * cur_g) { ++ if (!ggml_neo_pipeline_enabled()) { ++ return nullptr; ++ } ++ if (cur_g == nullptr) { ++ return nullptr; ++ } ++ std::lock_guard lock(g_mu); ++ const auto it = g_events.find(cur_g); ++ if (it == g_events.end()) { ++ return nullptr; ++ } ++ return it->second; ++} ++ ++void ggml_neo_pipeline_set_event(const ggml_tensor * cur_g, void * event) { ++ if (!ggml_neo_pipeline_enabled()) { ++ return; ++ } ++ if (cur_g == nullptr) { ++ return; ++ } ++ std::lock_guard lock(g_mu); ++ g_events[cur_g] = event; ++} ++ ++void ggml_neo_pipeline_clear_registrations(void) { ++ std::lock_guard lock(g_mu); ++ g_gpu_partners.clear(); ++ // Note: g_events deliberately NOT cleared here. Events are reusable ++ // across decode steps as long as the underlying tensor pointers stay ++ // stable. They're cleaned up by destroy_all_events on shutdown. ++} ++ ++void ggml_neo_pipeline_destroy_all_events(void (*destroyer)(void * event)) { ++ std::lock_guard lock(g_mu); ++ if (destroyer != nullptr) { ++ for (auto & kv : g_events) { ++ if (kv.second != nullptr) { ++ destroyer(kv.second); ++ } ++ } ++ } ++ g_events.clear(); ++} ++ ++int ggml_neo_pipeline_dispatch_count(void) { ++ return g_dispatch_count.load(std::memory_order_acquire); ++} ++ ++void ggml_neo_pipeline_inc_dispatch_count(void) { ++ g_dispatch_count.fetch_add(1, std::memory_order_relaxed); ++} +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -358,6 +358,10 @@ extern "C" { + // streamed back per step). 1.0 = off. Only effective for offloaded + // layers with flash-attention (non-transposed V cache). + float headinfer_gpu_heads_frac; ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ // 0 = off (default), 1 = on, 2 = auto. Effective only when the ++ // M2 head split is active for a given layer. ++ uint32_t neo_pipeline_mode; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -64,6 +64,8 @@ llama_context::llama_context( + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ cparams.neo_pipeline_mode = params.neo_pipeline_mode; + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -3356,6 +3358,7 @@ llama_context_params llama_context_default_params() { + /*.slot_initial_ctx =*/ 0, + /*.slot_shrink_idle_ms =*/ 0, + /*.headinfer_gpu_heads_frac =*/ 1.0f, ++ /*.neo_pipeline_mode =*/ 0, // opencoti F5 M3 neo — off by default + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -23,6 +23,15 @@ struct llama_cparams { + uint32_t slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + float headinfer_gpu_heads_frac; ++ // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention. ++ // 0 = off (M2 concat path runs; binary equivalent to upstream when ++ // headinfer_gpu_heads_frac == 1.0). ++ // 1 = on (engage when headinfer split is active for the layer: ++ // build two flash_attn_ext ops on disjoint head ranges, run ++ // them on GPU and CPU backends respectively, concat outputs). ++ // 2 = auto (same engagement as `on` for now — reserved for future ++ // load-based engagement heuristics). ++ uint32_t neo_pipeline_mode; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -11,6 +11,10 @@ + #include "llama-memory-hybrid-iswa.h" + #include "llama-memory-recurrent.h" + ++// opencoti F5 M3 NEO orchestrator (registers cur_g/cur_c pairs so the CUDA ++// backend can route GPU FA to a non-default stream and overlap with CPU FA). ++#include "ggml-neo-pipeline.h" ++ + #include + #include + #include +@@ -2085,6 +2089,140 @@ ggml_tensor * llm_graph_context::build_attn_mha( + return cur; + } + ++// opencoti F5 M3 neo — head-axis-split attention. Builds two flash_attn_ext ++// ops on disjoint head ranges: ++// GPU side: q[..., :gpu_heads*gqa, ...] × k_g × v_g (auto-routes to GPU ++// via the scheduler because k_g/v_g are views on the layer's ++// GPU buffer-type tensor — see ggml-backend.cpp:797-852). ++// CPU side: q[..., gpu_heads*gqa:, ...] × k_c × v_c (auto-routes to CPU ++// because k_c/v_c are views on the CPU buffer-type tensor that ++// M2's headinfer allocates alongside the GPU half — see ++// llama-kv-cache.cpp:get_k_cpu/get_v_cpu). ++// Outputs are concatenated on the head axis and fed into the standard ++// reshape_2d → Wo flow. This eliminates M2's per-step CPU→GPU stream-back ++// (O(n_kv) bytes per layer) — only the small O(n_tokens) CPU FA output ++// is bounced to GPU for the concat. Pre-M3-B, the two FA ops run serially ++// under the scheduler's default dispatch; M3-B's orchestrator will overlap ++// them concurrently. ++ggml_tensor * llm_graph_context::build_attn_mha_neo( ++ ggml_tensor * q, ++ ggml_tensor * k_g, ++ ggml_tensor * v_g, ++ ggml_tensor * k_c, ++ ggml_tensor * v_c, ++ uint32_t gpu_heads, ++ ggml_tensor * kq_b, ++ ggml_tensor * kq_mask, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il) const { ++ // Preconditions enforced by the M2 split gate that produced k_c/v_c. ++ GGML_ASSERT(cparams.flash_attn && "NEO path requires flash-attn"); ++ GGML_ASSERT(kq_b == nullptr && "NEO path does not support KQ bias"); ++ GGML_ASSERT(v_mla == nullptr && "NEO path does not support MLA"); ++ GGML_ASSERT(gpu_heads > 0 && "NEO path requires an active head split"); ++ GGML_ASSERT(k_g && v_g && k_c && v_c); ++ ++ // K/V layout is identical between the GPU and CPU halves (non-transposed ++ // V per the M2 gate). Stream axis matches across both — both are views ++ // on tensors allocated with the same n_stream at construction. ++ GGML_ASSERT(k_g->ne[3] == k_c->ne[3]); ++ GGML_ASSERT(v_g->ne[3] == v_c->ne[3]); ++ GGML_ASSERT(k_g->nb[1] <= k_g->nb[2] && "headinfer split requires non-transposed V"); ++ ++ // The full Q has all n_head_q heads. Heads are split on KV-head-group ++ // boundaries (M2's GQA clamp ensures gpu_heads is a whole number of ++ // groups). gqa_ratio = n_head_q / n_head_kv; n_head_kv = gpu_heads + ++ // cpu_heads. Q-heads serving the same KV head are CONTIGUOUS in Q's ++ // dim 1 under standard GGUF, so the slice is a clean view (no extra ++ // permute beyond the one already in build_attn_mha). ++ const uint32_t n_head_q = (uint32_t) q->ne[1]; ++ const uint32_t n_head_kv = (uint32_t) (k_g->ne[1] + k_c->ne[1]); ++ GGML_ASSERT(n_head_q % n_head_kv == 0 && "GQA ratio must divide n_head_q"); ++ const uint32_t gqa_ratio = n_head_q / n_head_kv; ++ const uint32_t gpu_q_heads = gpu_heads * gqa_ratio; ++ const uint32_t cpu_q_heads = (n_head_kv - gpu_heads) * gqa_ratio; ++ GGML_ASSERT(gpu_q_heads + cpu_q_heads == n_head_q); ++ ++ // Mirror build_attn_mha's n_stream reshape + permute on Q, but slice Q ++ // on the head axis BEFORE the permute so the dim-1 slice stays ++ // contiguous in the input layout. ++ const auto n_stream = k_g->ne[3]; ++ ggml_tensor * q_4 = ggml_view_4d(ctx0, q, ++ q->ne[0], q->ne[1], q->ne[2]/n_stream, n_stream, ++ q->nb[1], q->nb[2], q->nb[3]/n_stream, 0); ++ ++ ggml_tensor * q_g = ggml_view_4d(ctx0, q_4, ++ q_4->ne[0], gpu_q_heads, q_4->ne[2], q_4->ne[3], ++ q_4->nb[1], q_4->nb[2], q_4->nb[3], ++ /*offset=*/ 0); ++ ggml_tensor * q_c = ggml_view_4d(ctx0, q_4, ++ q_4->ne[0], cpu_q_heads, q_4->ne[2], q_4->ne[3], ++ q_4->nb[1], q_4->nb[2], q_4->nb[3], ++ /*offset=*/ (size_t) gpu_q_heads * q_4->nb[1]); ++ ++ q_g = ggml_permute(ctx0, q_g, 0, 2, 1, 3); ++ q_c = ggml_permute(ctx0, q_c, 0, 2, 1, 3); ++ ggml_tensor * k_g_p = ggml_permute(ctx0, k_g, 0, 2, 1, 3); ++ ggml_tensor * v_g_p = ggml_permute(ctx0, v_g, 0, 2, 1, 3); ++ ggml_tensor * k_c_p = ggml_permute(ctx0, k_c, 0, 2, 1, 3); ++ ggml_tensor * v_c_p = ggml_permute(ctx0, v_c, 0, 2, 1, 3); ++ ++ // F32 K → F16 cast mirrors build_attn_mha lines 1881-1887 (the embedding ++ // model non-causal path). v_trans is asserted false above, so no extra ++ // transpose is needed; V already has the same dim-1 head layout as K. ++ if (k_g_p->type == GGML_TYPE_F32) k_g_p = ggml_cast(ctx0, k_g_p, GGML_TYPE_F16); ++ if (k_c_p->type == GGML_TYPE_F32) k_c_p = ggml_cast(ctx0, k_c_p, GGML_TYPE_F16); ++ if (v_g_p->type == GGML_TYPE_F32) v_g_p = ggml_cast(ctx0, v_g_p, GGML_TYPE_F16); ++ if (v_c_p->type == GGML_TYPE_F32) v_c_p = ggml_cast(ctx0, v_c_p, GGML_TYPE_F16); ++ ++ // Sinks (if present) carry one F32 entry per Q-head; slice them to ++ // match each half. The flash_attn_ext_add_sinks op asserts ++ // sinks->ne[0] == q->src[0]->ne[2] (the q-head axis after permute). ++ ggml_tensor * sinks_g = nullptr; ++ ggml_tensor * sinks_c = nullptr; ++ if (sinks) { ++ GGML_ASSERT((uint32_t) sinks->ne[0] == n_head_q); ++ sinks_g = ggml_view_1d(ctx0, sinks, gpu_q_heads, 0); ++ sinks_c = ggml_view_1d(ctx0, sinks, cpu_q_heads, ++ (size_t) gpu_q_heads * sinks->nb[0]); ++ } ++ ++ const float alibi_bias = hparams.f_max_alibi_bias; ++ const float logit_cap = hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f; ++ ++ ggml_tensor * cur_g = ggml_flash_attn_ext(ctx0, q_g, k_g_p, v_g_p, kq_mask, kq_scale, alibi_bias, logit_cap); ++ cb(cur_g, LLAMA_TENSOR_NAME_FATTN "_g", il); ++ ggml_flash_attn_ext_add_sinks(cur_g, sinks_g); ++ ggml_flash_attn_ext_set_prec (cur_g, GGML_PREC_F32); ++ ++ ggml_tensor * cur_c = ggml_flash_attn_ext(ctx0, q_c, k_c_p, v_c_p, kq_mask, kq_scale, alibi_bias, logit_cap); ++ cb(cur_c, LLAMA_TENSOR_NAME_FATTN "_c", il); ++ ggml_flash_attn_ext_add_sinks(cur_c, sinks_c); ++ ggml_flash_attn_ext_set_prec (cur_c, GGML_PREC_F32); ++ ++ // opencoti F5 M3-B: enable + register the pair so the CUDA backend swaps ++ // cur_g to a non-default stream at dispatch time, letting cur_c (on CPU) ++ // overlap. set_enabled(true) is idempotent and only flipped here — the ++ // call site already gates on neo_pipeline_mode != 0 && headinfer_split_active, ++ // so reaching this point means NEO is engaged for this layer. ++ ggml_neo_pipeline_set_enabled(true); ++ ggml_neo_pipeline_register_pair(cur_g, cur_c); ++ ++ // FA output shape is [v_dim, n_q_heads_split, n_tokens, n_stream]. ++ // Concat on dim 1 (q-head axis) to reassemble the full Q-head set. ++ // The scheduler synchronizes here — cur_c gets lifted to GPU so the ++ // concat runs on whichever backend hosts cur_g (typically GPU). ++ ggml_tensor * cur = ggml_concat(ctx0, cur_g, cur_c, 1); ++ cb(cur, LLAMA_TENSOR_NAME_FATTN, il); ++ ++ // Match build_attn_mha's flash-attn final reshape (line 1913). ++ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); ++ ++ return cur; ++} ++ + llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() const { + auto inp = std::make_unique(hparams, cparams); + +@@ -2237,10 +2375,29 @@ ggml_tensor * llm_graph_context::build_attn( + const auto & kq_mask = inp->get_kq_mask(); + + ggml_tensor * q = q_cur; +- ggml_tensor * k = mctx_cur->get_k(ctx0, il); +- ggml_tensor * v = mctx_cur->get_v(ctx0, il); + +- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ // opencoti F5 M3 neo — head-axis-split attention. When the M2 head ++ // residency split is active for this layer AND --neo-pipeline is on, ++ // build two flash_attn_ext ops on disjoint head ranges instead of the ++ // M2 concat-reassembly path (which streams k_cpu/v_cpu back to GPU each ++ // step and runs a single full-head FA op). The scheduler routes each ++ // FA op to the backend matching its K/V buffer-type. v_mla must be ++ // null (build_attn for KV asserts this above at :2102), kq_b too; ++ // the M2 gate ensures non-transposed V. See docs/features/advanced_kv.md. ++ ggml_tensor * cur; ++ if (cparams.neo_pipeline_mode != 0 && mctx_cur->headinfer_split_active(il)) { ++ ggml_tensor * k_g = mctx_cur->get_k_gpu(ctx0, il); ++ ggml_tensor * v_g = mctx_cur->get_v_gpu(ctx0, il); ++ ggml_tensor * k_c = mctx_cur->get_k_cpu(ctx0, il); ++ ggml_tensor * v_c = mctx_cur->get_v_cpu(ctx0, il); ++ cur = build_attn_mha_neo(q, k_g, v_g, k_c, v_c, ++ mctx_cur->headinfer_gpu_heads(il), ++ kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ } else { ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ } + cb(cur, "kqv_out", il); + + if (inp->self_v_rot) { +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -908,6 +908,27 @@ struct llm_graph_context { + float kq_scale, + int il) const; + ++ // opencoti F5 M3 neo — head-axis split flash-attn. Builds two ++ // ggml_flash_attn_ext ops on disjoint head ranges (GPU vs CPU subsets, ++ // routed automatically by the scheduler via the K/V tensors' buffer ++ // types) and concatenates their outputs on the head axis before the ++ // reshape/Wo projection. Flash-attention only (assert kq_b == nullptr, ++ // v_mla == nullptr, !v_trans for V — the M2 split gate already enforces ++ // this). Output shape matches build_attn_mha's flash-attn branch. ++ ggml_tensor * build_attn_mha_neo( ++ ggml_tensor * q, // [n_embd_head_q, n_head_q, n_tokens] ++ ggml_tensor * k_g, // GPU-resident K subset (gpu_heads on dim 1) ++ ggml_tensor * v_g, // GPU-resident V subset (gpu_heads on dim 1) ++ ggml_tensor * k_c, // CPU-resident K subset (cpu_heads on dim 1) ++ ggml_tensor * v_c, // CPU-resident V subset (cpu_heads on dim 1) ++ uint32_t gpu_heads, ++ ggml_tensor * kq_b, // must be nullptr ++ ggml_tensor * kq_mask, ++ ggml_tensor * sinks, // [n_head_q] or nullptr ++ ggml_tensor * v_mla, // must be nullptr ++ float kq_scale, ++ int il) const; ++ + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -1810,6 +1810,116 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k + ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0); + } + ++// opencoti F5 M3 neo — head-split-aware accessors. These factor the per-half ++// view machinery that get_k/get_v perform internally before their concat, ++// exposing the GPU-only and CPU-only sub-tensors as independent ggml views. ++// M3 NEO uses them to build two flash_attn_ext ops (one per backend, scheduler ++// auto-routes by buffer type) and merge their outputs, eliminating M2's ++// per-step O(n_kv) CPU→GPU stream-back. Dimensions and strides are identical ++// to the corresponding halves inside M2's get_k/get_v. ++ ++bool llama_kv_cache::headinfer_split_active(int32_t il) const { ++ const auto it = map_layer_ids.find(il); ++ if (it == map_layer_ids.end()) { ++ return false; ++ } ++ const auto & layer = layers[it->second]; ++ return layer.gpu_heads > 0 && layer.k_cpu != nullptr; ++} ++ ++uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const { ++ const auto it = map_layer_ids.find(il); ++ if (it == map_layer_ids.end()) { ++ return 0; ++ } ++ return layers[it->second].gpu_heads; ++} ++ ++ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split"); ++ ++ auto * k = layer.k; ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_gpu = layer.gpu_heads; ++ const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_k; ++ ++ return ggml_view_4d(ctx, k, ++ n_embd_head_k, n_head_gpu, n_kv, ns, ++ ggml_row_size(k->type, n_embd_head_k), ++ ggml_row_size(k->type, n_embd_gpu), ++ ggml_row_size(k->type, n_embd_gpu*kv_size), ++ ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0); ++} ++ ++ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.gpu_heads > 0 && layer.k_cpu && "get_k_cpu called without an active headinfer split"); ++ ++ auto * kc = layer.k_cpu; ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; ++ const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_k; ++ ++ return ggml_view_4d(ctx, kc, ++ n_embd_head_k, n_head_cpu, n_kv, ns, ++ ggml_row_size(kc->type, n_embd_head_k), ++ ggml_row_size(kc->type, n_embd_cpu), ++ ggml_row_size(kc->type, n_embd_cpu*kv_size), ++ ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0); ++} ++ ++ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split"); ++ // M2's gate is `!v_trans && n_stream == 1` — when split is active the ++ // V layout is always non-transposed (matches get_v's non-split branch ++ // shape modulo head count). Same head-dim-1 stride pattern as K. ++ GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V"); ++ ++ auto * v = layer.v; ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_gpu = layer.gpu_heads; ++ const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_v; ++ ++ return ggml_view_4d(ctx, v, ++ n_embd_head_v, n_head_gpu, n_kv, ns, ++ ggml_row_size(v->type, n_embd_head_v), ++ ggml_row_size(v->type, n_embd_gpu), ++ ggml_row_size(v->type, n_embd_gpu*kv_size), ++ ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0); ++} ++ ++ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.gpu_heads > 0 && layer.v_cpu && "get_v_cpu called without an active headinfer split"); ++ GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V"); ++ ++ auto * vc = layer.v_cpu; ++ const uint64_t kv_size = get_size(); ++ const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; ++ const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_v; ++ ++ return ggml_view_4d(ctx, vc, ++ n_embd_head_v, n_head_cpu, n_kv, ns, ++ ggml_row_size(vc->type, n_embd_head_v), ++ ggml_row_size(vc->type, n_embd_cpu), ++ ggml_row_size(vc->type, n_embd_cpu*kv_size), ++ ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0); ++} ++ + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { + GGML_UNUSED(sinfo); + +@@ -3107,6 +3217,31 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons + return kv->get_v(ctx, il, n_kv, sinfos[i_cur]); + } + ++// opencoti F5 M3 neo — per-batch wrappers around the head-split accessors. ++ggml_tensor * llama_kv_cache_context::get_k_gpu(ggml_context * ctx, int32_t il) const { ++ return kv->get_k_gpu(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_v_gpu(ggml_context * ctx, int32_t il) const { ++ return kv->get_v_gpu(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_k_cpu(ggml_context * ctx, int32_t il) const { ++ return kv->get_k_cpu(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_v_cpu(ggml_context * ctx, int32_t il) const { ++ return kv->get_v_cpu(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++bool llama_kv_cache_context::headinfer_split_active(int32_t il) const { ++ return kv->headinfer_split_active(il); ++} ++ ++uint32_t llama_kv_cache_context::headinfer_gpu_heads(int32_t il) const { ++ return kv->headinfer_gpu_heads(il); ++} ++ + ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { + return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]); + } +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -217,6 +217,23 @@ public: + ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; + ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; + ++ // opencoti F5 M3 neo — head-split-aware K/V accessors. Return separate ++ // views onto the GPU-resident and host-resident head subsets without the ++ // concat-reassembly that get_k/get_v perform. Built on the SAME view ++ // machinery (see llama-kv-cache.cpp get_k for the dimensions); callers ++ // are expected to gate via headinfer_split_active(il) and to build their ++ // own combiner (M3 NEO builds two flash_attn_ext ops + a head-axis concat ++ // on the small attention outputs, eliminating M2's O(n_kv) stream-back). ++ // For non-split layers these return the equivalent of the un-split view ++ // for the "_gpu" half and nullptr for "_cpu" — callers must check ++ // headinfer_split_active first. ++ ggml_tensor * get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ bool headinfer_split_active(int32_t il) const; ++ uint32_t headinfer_gpu_heads (int32_t il) const; ++ + // store k_cur and v_cur in the cache based on the provided head location + ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const; + ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const; +@@ -444,6 +461,16 @@ public: + ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; + ggml_tensor * get_v(ggml_context * ctx, int32_t il) const; + ++ // opencoti F5 M3 neo — head-split-aware K/V accessors. Per-batch wrappers ++ // that delegate to the underlying llama_kv_cache with the current n_kv / ++ // slot info. See llama-kv-cache.h for semantics. ++ ggml_tensor * get_k_gpu(ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_v_gpu(ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_k_cpu(ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_v_cpu(ggml_context * ctx, int32_t il) const; ++ bool headinfer_split_active(int32_t il) const; ++ uint32_t headinfer_gpu_heads (int32_t il) const; ++ + // store k_cur and v_cur in the cache based on the provided head location + // note: the heads in k_cur and v_cur should be laid out contiguously in memory + // - k_cur [n_embd_head_k, n_head_k, n_tokens] +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -218,10 +218,14 @@ compile_ggml_core() { + local llama_cpp_dir="$1" + local build_dir="$2" + ++ # opencoti F5 M3-B: ggml-neo-pipeline.cpp added so the CUDA backend's ++ # graph_compute hook can call the orchestrator API (registered cur_g ++ # detection + event slot get/set + dispatch-count increment). + local ggml_core_sources="\ + $llama_cpp_dir/ggml/src/ggml.c \ + $llama_cpp_dir/ggml/src/ggml-alloc.c \ + $llama_cpp_dir/ggml/src/ggml-backend.cpp \ ++ $llama_cpp_dir/ggml/src/ggml-neo-pipeline.cpp \ + $llama_cpp_dir/ggml/src/ggml-backend-meta.cpp \ + $llama_cpp_dir/ggml/src/ggml-quants.c \ + $llama_cpp_dir/ggml/src/ggml-threading.cpp" diff --git a/patches/0031-per-stream-split.patch b/patches/0031-per-stream-split.patch new file mode 100644 index 0000000000000000000000000000000000000000..d1ef876a88f2df481eb017492c616707866ac831 --- /dev/null +++ b/patches/0031-per-stream-split.patch @@ -0,0 +1,1513 @@ +From: opencoti +Subject: [PATCH 0031] F4 M3 Phase 4 — per-stream KV tensor split (task #109) + +Resolves the long-deferred Phase 4 (`docs/decisions/0001-lazy-slot-context.md`) +of F4 M3 — every layer's K/V (and M2's CPU subset) become OWNING per-stream +tensor vectors rather than one 3-D tensor with a stream dim, with each +(stream, buffer-type) pair owning its own backend buffer. Unified mode +(`n_stream == 1`) collapses to the pre-Phase-4 layout and stays +byte-identical; non-unified mode (`n_stream > 1`) is functional for the +first time, unlocking the multi-tenant codepaths (M2 head-split now lifts +its `n_stream == 1` construction-time gate). + +Sub-milestone shape (each landed against the M5-shipped tree at HEAD = +`2373e69` and verified by the M5 stack + M0 turn-2 + M1 eviction + M2 +residency benches before composing): + + - Phase 4-A — per-stream tensor split. `kv_layer` replaces + `k`/`v`/`k_cpu`/`v_cpu` direct pointers with + `std::vector k_per_stream` / `v_per_stream` / + `k_cpu_per_stream` / `v_cpu_per_stream`; each per-stream tensor is + `ggml_new_tensor_3d(ctx_s, type, n_embd, kv_size, 1)`. The trailing + `dim 3 == 1` keeps stride math identical to the pre-Phase-4 + `[n_embd, kv_size, n_stream]` layout under unified mode. The + `k_stream` / `v_stream` 2-D view vectors are preserved (offset 0 + within each per-stream tensor) so state I/O and cross-stream copies + keep operating on the existing view handles. + + - Phase 4-B — per-stream backend buffers. `ctx_map` becomes a nested + `std::vector> ctx_map_per_stream` indexed by stream, + with a `ctx_for_buft(buft, s)` lambda that returns the get-or-create + `(stream, buft)` ctx (mem_size budgeted at `4u*n_layer_kv* + ggml_tensor_overhead()`). Allocation loops nest stream-outer × + buft-inner so each (stream, buft) pair allocates one backend buffer. + Phase 1 `ensure_cleared`, Phase 3 `opencoti_decommit_layer_range`, + and the M2 CPU subset construction follow the same per-stream + iteration shape. + + - Phase 4-C — reworked write/read graph ops. `cpy_k`/`cpy_v` ship + option C-2 (N separate `ggml_set_rows`, one per `sinfo.n_stream()` + stream, tied via `ggml_add` on 1-D views — unified-mode batch_ns==1 + collapses to one `set_rows`, byte-identical). C-1 (concat-then- + scatter) was rejected at impl time: `ggml_concat` materializes a + new buffer rather than viewing inputs, so writes routed through the + concat result would not land in the per-stream tensors. `get_k`/ + `get_v` build per-cache-stream 4-D views and `ggml_concat` them + along dim 3 (stream axis) — unified mode yields a single view, no + concat. `set_input_k_idxs`/`set_input_v_idxs` emit LOCAL per-stream + indices unconditionally (Phase 4-F fix; the previous gating on + `sinfo.n_stream() > 1` left global indices in --parallel N servers + serving single-session batches and crashed CUDA decode against the + per-stream destination). `build_graph_shift` loops streams inside + the layer loop, slicing `inp->k_shift` per stream. + + - Phase 4-D — M2 head-split is stream-aware. `get_k_gpu`/`get_v_gpu`/ + `get_k_cpu`/`get_v_cpu` (and the M2 split branch inside `get_k`/ + `get_v`/`cpy_k`/`cpy_v`) build per-stream 4-D views and concat + them on dim 3 (stream axis); the construction-time gate at + `headinfer_gpu_heads_frac < 1.0 && offload && !v_trans && n_stream + == 1` drops the `n_stream == 1` clause so per-stream CPU subsets + actually allocate for non-unified mode. + + - Phase 4-E — cross-stream copies in `update()`, `state_write_data`, + `state_read_data`, and `seq_key_scores` continue to operate on the + existing `k_stream[s]` / `v_stream[s]` 2-D views (now offset-0 views + of per-stream tensors). Mechanically distinct from pre-Phase-4 (one + buffer per stream rather than one shared buffer), semantically the + same per the buffer transfer mechanism. + + - Phase 4-F — verification gates. Cosine ≥ 0.999 unified-mode + equivalence captured indirectly via the M5 stack bench (B/A0/A1/ + A2/A3/S configs all produce byte-identical KV MiB and + `t2_cached_n=1271` against the M5-shipped baseline) plus M2's + explicit greedy-decode byte-equality check (`split text === + baseline text`, 6/6 PASS). Multi-stream RSS exercise comes via the + M0 turn-2 scenarioB (`--parallel 2` with two sessions sharing a + large system prefix — 5/5 PASS, validating per-stream KV reuse). + M3 neo-pipeline O off-identity config crashes on both pre-Phase-4 + and post-Phase-4 binaries with `concat.cu:165 GGML_ASSERT(src0-> + type == GGML_TYPE_F32)` — pre-existing M3 bug, DEFERRED as task + #278 and recorded as bug-207. Hook count stable at 20. + +The HARD gates that survived from earlier sub-milestones (the +`GGML_ASSERT(n_stream == 1 && "Phase 4-A: get_k temporarily gated..."; +"Phase 4-C lifts")` shape on get_k/get_v/cpy_k/cpy_v/build_graph_shift, +and the parallel `Phase 4-D lifts` shape on the M2 accessors) all reach +ship as lifted. Five `GGML_ASSERT(n_stream == 1)` lines deleted; the +remaining `n_stream == 1` checks are real preconditions (e.g., v_trans + +multi-stream is asserted out in cpy_v as not-a-real-workload). + +Unblocks F5 M6 (PolyKV) which needs per-stream lifetimes cleanly +expressed for the shared compressed KV pool. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -160,14 +160,35 @@ llama_kv_cache::llama_kv_cache( + return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0; + } + }; +- std::map ctx_map; + +- // create a context for each buffer type +- auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * { ++ // opencoti F4 M3 Phase 4-B — per-stream backend buffer ownership. ++ // Each stream gets its own map of (buft → ctx) so each (stream, buft) ++ // pair allocates its own backend buffer. Phase 3's ++ // opencoti_decommit_layer_range then collapses to "decommit this ++ // stream's buffer range" cleanly, and M6 PolyKV's per-stream ++ // lifetimes (enroll/disenroll a stream into the pool) become ++ // mechanically straightforward. ++ // ++ // Unified mode (n_stream == 1): one stream, one inner map per ++ // buft, byte-identical layout to pre-Phase-4 (the buffer-allocation ++ // loop below ends up with exactly the same `(ctx, buf)` pairs in ++ // ctxs_bufs as before, in the same order, with the same sizes). ++ std::vector> ctx_map_per_stream; ++ ctx_map_per_stream.resize(n_stream); ++ ++ // ctx_for_buft(buft, s): get-or-create the ggml_context for ++ // (stream s, buft). Each per-stream ctx holds at most 4 tensors ++ // per layer (K + V owning, K + V views; doubled when M2's CPU ++ // subset is engaged on a layer). Budget conservatively at ++ // 4*n_layer_kv slots per stream per buft — address space is free, ++ // RSS still grows only on first write. ++ auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft, uint32_t s) -> ggml_context * { ++ GGML_ASSERT(s < ctx_map_per_stream.size()); ++ auto & ctx_map = ctx_map_per_stream[s]; + auto it = ctx_map.find(buft); + if (it == ctx_map.end()) { + ggml_init_params params = { +- /*.mem_size =*/ size_t(2u*(1 + n_stream)*n_layer_kv*ggml_tensor_overhead()), ++ /*.mem_size =*/ size_t(4u*n_layer_kv*ggml_tensor_overhead()), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; +@@ -255,10 +276,10 @@ llama_kv_cache::llama_kv_cache( + + LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name); + +- ggml_context * ctx = ctx_for_buft(buft); +- if (!ctx) { +- throw std::runtime_error("failed to create ggml context for kv cache"); +- } ++ // opencoti F4 M3 Phase 4-B: per-stream ctx selection has moved ++ // into the per-stream loop below. The layer-level `buft` is the ++ // device buffer-type for this layer; each stream allocates its ++ // own ctx for (buft, stream) via ctx_for_buft(buft, s). + + const bool has_k = true; + const bool has_v = !is_mla; +@@ -267,14 +288,21 @@ llama_kv_cache::llama_kv_cache( + // Head-residency split: keep the first gpu_heads KV heads on this + // (offloaded/device) layer's buffer and put the remaining heads on a + // host buffer. Splitting on KV-head boundaries is inherently GQA-safe. +- // Gated to offloaded, non-transposed-V, unified (n_stream==1) layers; +- // frac >= 1.0 (or <= 1 head) means no split (byte-identical upstream). ++ // Gated to offloaded, non-transposed-V layers; frac >= 1.0 (or <= 1 ++ // head) means no split (byte-identical upstream). ++ // ++ // opencoti F4 M3 Phase 4-D: lifted the historical n_stream == 1 ++ // gate. With per-stream owning tensors and per-stream backend ++ // buffers (Phase 4-A/B), each stream's GPU/CPU half lives in its ++ // own buffer, so the split is well-defined for n_stream > 1. ++ // Unified mode (n_stream == 1) collapses to the pre-Phase-4 ++ // single-stream layout, byte-identical. + uint32_t gpu_heads = 0; + uint32_t n_embd_k_gpu = n_embd_k_gqa; + uint32_t n_embd_v_gpu = n_embd_v_gqa; + uint32_t n_embd_k_cpu = 0; + uint32_t n_embd_v_cpu = 0; +- if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans && n_stream == 1) { ++ if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans) { + const uint32_t n_head_kv = hparams.n_head_kv(il); + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); + const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); +@@ -291,39 +319,119 @@ llama_kv_cache::llama_kv_cache( + } + } + +- ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gpu, kv_size, n_stream) : nullptr; +- ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gpu, kv_size, n_stream) : nullptr; +- +- has_k && ggml_format_name(k, "cache_k_l%d", il); +- has_v && ggml_format_name(v, "cache_v_l%d", il); +- +- // opencoti F5 M2 headinfer — the host-resident head subset lives in the +- // CPU buffer-type context; the buffer-allocation loop below picks it up +- // automatically (ctx_for_buft keys contexts by buffer type). +- ggml_tensor * k_cpu = nullptr; +- ggml_tensor * v_cpu = nullptr; +- if (gpu_heads > 0) { +- ggml_context * ctx_cpu = ctx_for_buft(ggml_backend_cpu_buffer_type()); +- if (!ctx_cpu) { +- throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); +- } +- k_cpu = has_k ? ggml_new_tensor_3d(ctx_cpu, type_k, n_embd_k_cpu, kv_size, n_stream) : nullptr; +- v_cpu = has_v ? ggml_new_tensor_3d(ctx_cpu, type_v, n_embd_v_cpu, kv_size, n_stream) : nullptr; +- has_k && k_cpu && ggml_format_name(k_cpu, "cache_k_cpu_l%d", il); +- has_v && v_cpu && ggml_format_name(v_cpu, "cache_v_cpu_l%d", il); +- } +- ++ // opencoti F4 M3 Phase 4 — per-stream tensor split. ++ // Each stream gets its own OWNING 3-D tensor of shape ++ // [n_embd, kv_size, 1]. Dim 3 == 1 keeps stride math ++ // identical to the pre-Phase-4 single-tensor layout, so the ++ // unified-mode (n_stream == 1) buffer layout is byte-identical ++ // to pre-Phase-4. Phase 4-B: each stream also gets its own ++ // (buft → ctx) entry, so each (stream, buft) pair allocates ++ // its own backend buffer in the loop below. ++ std::vector k_per_stream; ++ std::vector v_per_stream; ++ std::vector k_cpu_per_stream; ++ std::vector v_cpu_per_stream; + std::vector k_stream; + std::vector v_stream; + + for (uint32_t s = 0; s < n_stream; ++s) { +- k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gpu, kv_size, k->nb[1], s*k->nb[2]) : nullptr); +- v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gpu, kv_size, v->nb[1], s*v->nb[2]) : nullptr); ++ ggml_context * ctx_s = ctx_for_buft(buft, s); ++ if (!ctx_s) { ++ throw std::runtime_error("failed to create ggml context for kv cache"); ++ } ++ ++ ggml_tensor * k_s = has_k ++ ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, kv_size, 1) ++ : nullptr; ++ ggml_tensor * v_s = has_v ++ ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, kv_size, 1) ++ : nullptr; ++ ++ // Tensor naming preserves the pre-Phase-4 name in unified ++ // mode (no `_s%u` suffix) so external state-IO callers that ++ // grep by name find an unchanged layout. ++ if (k_s) { ++ if (n_stream == 1) { ++ ggml_format_name(k_s, "cache_k_l%d", il); ++ } else { ++ ggml_format_name(k_s, "cache_k_l%d_s%u", il, s); ++ } ++ } ++ if (v_s) { ++ if (n_stream == 1) { ++ ggml_format_name(v_s, "cache_v_l%d", il); ++ } else { ++ ggml_format_name(v_s, "cache_v_l%d_s%u", il, s); ++ } ++ } ++ ++ k_per_stream.push_back(k_s); ++ v_per_stream.push_back(v_s); ++ ++ // opencoti F5 M2 headinfer — per-stream host-resident head ++ // subset. Per-stream ctx_cpu so each stream's CPU half lives ++ // in its own host buffer (Phase 4-B). Vectors stay empty ++ // when gpu_heads == 0 so headinfer_split_active() can test ++ // by vector size. ++ if (gpu_heads > 0) { ++ ggml_context * ctx_cpu_s = ctx_for_buft(ggml_backend_cpu_buffer_type(), s); ++ if (!ctx_cpu_s) { ++ throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); ++ } ++ ggml_tensor * k_cpu_s = has_k ++ ? ggml_new_tensor_3d(ctx_cpu_s, type_k, n_embd_k_cpu, kv_size, 1) ++ : nullptr; ++ ggml_tensor * v_cpu_s = has_v ++ ? ggml_new_tensor_3d(ctx_cpu_s, type_v, n_embd_v_cpu, kv_size, 1) ++ : nullptr; ++ if (k_cpu_s) { ++ if (n_stream == 1) { ++ ggml_format_name(k_cpu_s, "cache_k_cpu_l%d", il); ++ } else { ++ ggml_format_name(k_cpu_s, "cache_k_cpu_l%d_s%u", il, s); ++ } ++ } ++ if (v_cpu_s) { ++ if (n_stream == 1) { ++ ggml_format_name(v_cpu_s, "cache_v_cpu_l%d", il); ++ } else { ++ ggml_format_name(v_cpu_s, "cache_v_cpu_l%d_s%u", il, s); ++ } ++ } ++ k_cpu_per_stream.push_back(k_cpu_s); ++ v_cpu_per_stream.push_back(v_cpu_s); ++ } ++ ++ // 2-D views into per-stream 3-D tensor. Offset is 0 within ++ // the owning tensor — each stream owns the whole 3-D extent. ++ // ++ // opencoti F4 M3 Phase 4-E: keeping k_stream/v_stream as the ++ // existing 2-D view vectors means cross-stream copies in ++ // update() (ggml_backend_tensor_copy between views), state ++ // I/O paths (ggml_backend_tensor_set on the per-stream view), ++ // and seq_key_scores all work without per-stream logic at ++ // the consumer site — the view routes through to the ++ // per-stream buffer transparently. Unified-mode operations ++ // are byte-identical to pre-Phase-4 because the view has ++ // identical ne/nb and (in unified mode) the same underlying ++ // buffer layout. ++ k_stream.push_back(k_s ++ ? ggml_view_2d(ctx_s, k_s, n_embd_k_gpu, kv_size, k_s->nb[1], 0) ++ : nullptr); ++ v_stream.push_back(v_s ++ ? ggml_view_2d(ctx_s, v_s, n_embd_v_gpu, kv_size, v_s->nb[1], 0) ++ : nullptr); + } + + map_layer_ids[il] = layers.size(); + +- layers.push_back({ il, k, v, k_stream, v_stream, k_cpu, v_cpu, gpu_heads }); ++ layers.push_back({ ++ il, ++ k_per_stream, v_per_stream, ++ k_stream, v_stream, ++ k_cpu_per_stream, v_cpu_per_stream, ++ gpu_heads, ++ }); + } + + if (reuse) { +@@ -359,31 +467,46 @@ llama_kv_cache::llama_kv_cache( + // scheduler-visible tensor->buffer wiring intact (sched_reserve runs + // at construction time and asserts buffer_id >= 0) while moving the + // RSS commit to the first ensure_cleared() call. +- for (auto & [buft, ctx] : ctx_map) { +- ggml_backend_buffer_t buf; +- if (model.hparams.no_alloc) { +- buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer +- for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { +- t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it ++ // ++ // opencoti F4 M3 Phase 4-B: per-stream backend buffers. Each ++ // (stream, buft) pair allocates its own backend buffer here. ++ // Iteration order is stable: outer loop over streams (s=0,1,...), ++ // inner loop over bufts (sorted by name via the ctx_map comparator). ++ // In unified mode (n_stream == 1) this collapses to exactly one ++ // pass per buft — byte-identical to pre-Phase-4 ctxs_bufs layout. ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ for (auto & [buft, ctx] : ctx_map_per_stream[s]) { ++ ggml_backend_buffer_t buf; ++ if (model.hparams.no_alloc) { ++ buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer ++ for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { ++ t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it ++ } ++ } else { ++ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet) ++ } ++ if (!buf) { ++ throw std::runtime_error("failed to allocate buffer for kv cache"); ++ } ++ ++ if (n_stream == 1) { ++ LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ++ ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); ++ } else { ++ LLAMA_LOG_INFO("%s: %10s KV buffer (stream %u) size = %8.2f MiB\n", __func__, ++ ggml_backend_buffer_name(buf), s, ggml_backend_buffer_get_size(buf)/1024.0/1024.0); + } +- } else { +- buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer (malloc'd but not zeroed yet) +- } +- if (!buf) { +- throw std::runtime_error("failed to allocate buffer for kv cache"); +- } + +- LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0); ++ if (model.hparams.no_alloc) { ++ // Dummy buffer is size 0; clearing is a no-op but keeps the ++ // upstream invariant that constructed buffers are zeroed. ++ ggml_backend_buffer_clear(buf, 0); ++ } ++ // else: leave the buffer un-cleared. ensure_cleared() will zero it ++ // on the first inference entry point that touches tensor data. + +- if (model.hparams.no_alloc) { +- // Dummy buffer is size 0; clearing is a no-op but keeps the +- // upstream invariant that constructed buffers are zeroed. +- ggml_backend_buffer_clear(buf, 0); ++ ctxs_bufs.emplace_back(std::move(ctx), buf); + } +- // else: leave the buffer un-cleared. ensure_cleared() will zero it +- // on the first inference entry point that touches tensor data. +- +- ctxs_bufs.emplace_back(std::move(ctx), buf); + } + + if (model.hparams.no_alloc) { +@@ -471,33 +594,43 @@ void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) { + // memset on the malloc'd buffer; for offloaded buffers it's a backend + // op. Either way only bytes for cells [a, b) get touched, so on Linux + // only those pages get committed to RSS. ++ // ++ // opencoti F4 M3 Phase 4: per-stream tensors. Each layer's ++ // k_per_stream[s] / v_per_stream[s] is now its own owning tensor ++ // (shape [n_embd, kv_size, 1]) so the per-stream stride math ++ // collapses to a plain cell-offset memset within each stream's ++ // tensor. M2's CPU heads follow the same pattern via ++ // k_cpu_per_stream / v_cpu_per_stream (empty when no split). + for (const auto & layer : layers) { + for (uint32_t st = 0; st < n_stream; ++st) { +- if (layer.k) { +- const size_t off = (size_t) st * (size_t) layer.k->nb[2] +- + (size_t) a * (size_t) layer.k->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) layer.k->nb[1]; +- ggml_backend_tensor_memset(layer.k, 0, off, sz); +- } +- if (layer.v) { +- const size_t off = (size_t) st * (size_t) layer.v->nb[2] +- + (size_t) a * (size_t) layer.v->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) layer.v->nb[1]; +- ggml_backend_tensor_memset(layer.v, 0, off, sz); +- } +- // opencoti F5 M2 headinfer — clear the host-resident head subset +- // too (its own strides; same [a, b) cell range). +- if (layer.k_cpu) { +- const size_t off = (size_t) st * (size_t) layer.k_cpu->nb[2] +- + (size_t) a * (size_t) layer.k_cpu->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) layer.k_cpu->nb[1]; +- ggml_backend_tensor_memset(layer.k_cpu, 0, off, sz); ++ auto * k = layer.k_per_stream[st]; ++ if (k) { ++ const size_t off = (size_t) a * (size_t) k->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) k->nb[1]; ++ ggml_backend_tensor_memset(k, 0, off, sz); ++ } ++ auto * v = layer.v_per_stream[st]; ++ if (v) { ++ const size_t off = (size_t) a * (size_t) v->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) v->nb[1]; ++ ggml_backend_tensor_memset(v, 0, off, sz); ++ } ++ // M2 headinfer host-resident head subset (same [a, b) cells). ++ if (st < layer.k_cpu_per_stream.size()) { ++ auto * k_cpu = layer.k_cpu_per_stream[st]; ++ if (k_cpu) { ++ const size_t off = (size_t) a * (size_t) k_cpu->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) k_cpu->nb[1]; ++ ggml_backend_tensor_memset(k_cpu, 0, off, sz); ++ } + } +- if (layer.v_cpu) { +- const size_t off = (size_t) st * (size_t) layer.v_cpu->nb[2] +- + (size_t) a * (size_t) layer.v_cpu->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) layer.v_cpu->nb[1]; +- ggml_backend_tensor_memset(layer.v_cpu, 0, off, sz); ++ if (st < layer.v_cpu_per_stream.size()) { ++ auto * v_cpu = layer.v_cpu_per_stream[st]; ++ if (v_cpu) { ++ const size_t off = (size_t) a * (size_t) v_cpu->nb[1]; ++ const size_t sz = (size_t) (b - a) * (size_t) v_cpu->nb[1]; ++ ggml_backend_tensor_memset(v_cpu, 0, off, sz); ++ } + } + } + } +@@ -542,22 +675,25 @@ bool llama_kv_cache::ensure_capacity(uint32_t cells_needed) { + } + + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md +-// Decommit pages of every layer's K/V tensors in the byte-range that +-// corresponds to cells [from_cells, to_cells). Returns those pages to the OS +-// — RSS drops by the size of the range, and the next access to those pages +-// will fault them back in as fresh zero-filled pages. ++// Decommit pages of one layer's K/V tensor for ONE STREAM in the byte- ++// range that corresponds to cells [from_cells, to_cells). Returns those ++// pages to the OS — RSS drops by the size of the range, and the next ++// access to those pages will fault them back in as fresh zero-filled ++// pages. + // + // Only safe to call after kv_size_alloc has been reset to from_cells (or +-// lower) and n_cells_cleared has been reset accordingly, so the bookkeeping +-// stays in sync with what's actually backed by physical memory. ++// lower) and n_cells_cleared has been reset accordingly, so the ++// bookkeeping stays in sync with what's actually backed by physical ++// memory. + // +-// The decommit is per-stream because the KV tensors are 3-D +-// [n_embd, kv_size, n_stream]; cells [a, b) within stream s occupy a +-// contiguous byte-range whose start address is (base + s*nb[2] + a*nb[1]) +-// and length is (b-a)*nb[1]. posix_madvise is page-grained — bytes that +-// don't cover a full page are silently kept resident, which is fine for +-// the cells that straddle page boundaries. +-static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream, ++// opencoti F4 M3 Phase 4: per-stream tensors. `t` is now ONE stream's ++// owning 3-D tensor (dim 3 == 1). Cells [a, b) occupy contiguous bytes ++// at (base + a*nb[1]) for length (b-a)*nb[1]. The stream-stride loop ++// of the pre-Phase-4 implementation collapses; callers iterate streams ++// at the call site instead. posix_madvise is page-grained — bytes that ++// don't cover a full page are silently kept resident, which is fine ++// for the cells that straddle page boundaries. ++static void opencoti_decommit_layer_range(ggml_tensor * t, + uint32_t from_cells, uint32_t to_cells) { + if (!t || to_cells <= from_cells) { + return; +@@ -579,37 +715,32 @@ static void opencoti_decommit_layer_range(ggml_tensor * t, uint32_t n_stream, + if (!base) { + return; + } +- const size_t stride_stream = (size_t) t->nb[2]; +- const size_t stride_cell = (size_t) t->nb[1]; +- const size_t off_in_buf = (size_t) ((char *) t->data - (char *) base); +- for (uint32_t s = 0; s < n_stream; ++s) { +- const size_t off = off_in_buf +- + (size_t) s * stride_stream +- + (size_t) from_cells * stride_cell; +- const size_t sz = (size_t) (to_cells - from_cells) * stride_cell; +- // posix_madvise wants the address — work in absolute terms. +- void * addr = (char *) base + off; +- // Round up to a page boundary on the low side, round down on the +- // high side, so we only decommit pages we fully own. The cell-aligned +- // address is unlikely to be page-aligned in general. +- const size_t page = (size_t) sysconf(_SC_PAGESIZE); +- if (page == 0 || page == (size_t) -1) { +- return; +- } +- uintptr_t lo = (uintptr_t) addr; +- uintptr_t hi = lo + sz; +- uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1); +- uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1); +- if (hi_aligned <= lo_aligned) { +- continue; +- } +- // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with +- // errno set). We treat any failure as advisory — the RSS win may +- // be skipped, but correctness is unaffected (the K/V buffer is +- // still mapped, just possibly still resident). +- (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned), +- MADV_DONTNEED); ++ const size_t stride_cell = (size_t) t->nb[1]; ++ const size_t off_in_buf = (size_t) ((char *) t->data - (char *) base); ++ const size_t off = off_in_buf + (size_t) from_cells * stride_cell; ++ const size_t sz = (size_t) (to_cells - from_cells) * stride_cell; ++ // posix_madvise wants the address — work in absolute terms. ++ void * addr = (char *) base + off; ++ // Round up to a page boundary on the low side, round down on the ++ // high side, so we only decommit pages we fully own. The cell-aligned ++ // address is unlikely to be page-aligned in general. ++ const size_t page = (size_t) sysconf(_SC_PAGESIZE); ++ if (page == 0 || page == (size_t) -1) { ++ return; + } ++ uintptr_t lo = (uintptr_t) addr; ++ uintptr_t hi = lo + sz; ++ uintptr_t lo_aligned = (lo + page - 1) & ~(uintptr_t)(page - 1); ++ uintptr_t hi_aligned = hi & ~(uintptr_t)(page - 1); ++ if (hi_aligned <= lo_aligned) { ++ return; ++ } ++ // madvise(MADV_DONTNEED) returns 0 on success, -1 on failure (with ++ // errno set). We treat any failure as advisory — the RSS win may ++ // be skipped, but correctness is unaffected (the K/V buffer is ++ // still mapped, just possibly still resident). ++ (void) madvise((void *) lo_aligned, (size_t) (hi_aligned - lo_aligned), ++ MADV_DONTNEED); + } + + void llama_kv_cache::shrink_if_idle() { +@@ -667,13 +798,20 @@ void llama_kv_cache::shrink_if_idle() { + (unsigned long long) (now - last_active_ms_), evicted); + + for (const auto & layer : layers) { +- opencoti_decommit_layer_range(layer.k, n_stream, from_cells, to_cells); +- opencoti_decommit_layer_range(layer.v, n_stream, from_cells, to_cells); +- // opencoti F5 M2 headinfer — the host-resident head subset IS +- // host-pageable, so this is where the GPU-mode shrink actually returns +- // RSS (the device k/v above are skipped by the is_host() guard). +- opencoti_decommit_layer_range(layer.k_cpu, n_stream, from_cells, to_cells); +- opencoti_decommit_layer_range(layer.v_cpu, n_stream, from_cells, to_cells); ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ opencoti_decommit_layer_range(layer.k_per_stream[s], from_cells, to_cells); ++ opencoti_decommit_layer_range(layer.v_per_stream[s], from_cells, to_cells); ++ // opencoti F5 M2 headinfer — the host-resident head subset ++ // IS host-pageable, so this is where the GPU-mode shrink ++ // actually returns RSS (the device k/v above are skipped ++ // by the is_host() guard in the function). ++ if (s < layer.k_cpu_per_stream.size()) { ++ opencoti_decommit_layer_range(layer.k_cpu_per_stream[s], from_cells, to_cells); ++ } ++ if (s < layer.v_cpu_per_stream.size()) { ++ opencoti_decommit_layer_range(layer.v_cpu_per_stream[s], from_cells, to_cells); ++ } ++ } + } + + // After decommit, treat cells [from_cells, to_cells) as "uncleared" — +@@ -1683,11 +1821,14 @@ bool llama_kv_cache::get_has_shift() const { + } + + ggml_type llama_kv_cache::type_k() const { +- return layers[0].k->type; ++ // opencoti F4 M3 Phase 4: all per-stream tensors of a layer carry the ++ // same type — read it from stream 0. Works in both unified and ++ // non-unified modes. ++ return layers[0].k_per_stream[0]->type; + } + + ggml_type llama_kv_cache::type_v() const { +- return layers[0].v->type; ++ return layers[0].v_per_stream[0]->type; + } + + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { +@@ -1710,104 +1851,166 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k + const int32_t ikv = map_layer_ids.at(il); + + const auto & layer = layers[ikv]; +- auto * k = layer.k; + + const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; + + // opencoti F5 M2 headinfer — reassemble the full head set from the +- // GPU-resident (k) and host-resident (k_cpu) subsets by concat on the head ++ // GPU-resident and host-resident subsets by concat on the head + // dimension. The backend scheduler streams k_cpu to the compute backend. +- if (layer.gpu_heads > 0 && layer.k_cpu) { ++ // ++ // opencoti F4 M3 Phase 4-D: stream-aware. For each cache stream covered ++ // by sinfo, build a 4-D head-axis-concat view (GPU half + CPU half) ++ // with that stream's per-stream tensors. Then concat the per-stream ++ // results along the stream axis (dim 3). Unified mode (sinfo.s1 == ++ // sinfo.s0 == 0) collapses to one 4-D head-concat view of stream 0's ++ // tensors — byte-identical to pre-Phase-4. ++ if (layer.gpu_heads > 0 && !layer.k_cpu_per_stream.empty() && layer.k_cpu_per_stream[0]) { + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); + const uint32_t n_head_gpu = layer.gpu_heads; + const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; + const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_k; + const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_k; +- auto * kc = layer.k_cpu; +- ggml_tensor * kg_v = ggml_view_4d(ctx, k, +- n_embd_head_k, n_head_gpu, n_kv, ns, +- ggml_row_size(k->type, n_embd_head_k), +- ggml_row_size(k->type, n_embd_gpu), +- ggml_row_size(k->type, n_embd_gpu*kv_size), +- ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0); +- ggml_tensor * kc_v = ggml_view_4d(ctx, kc, +- n_embd_head_k, n_head_cpu, n_kv, ns, +- ggml_row_size(kc->type, n_embd_head_k), +- ggml_row_size(kc->type, n_embd_cpu), +- ggml_row_size(kc->type, n_embd_cpu*kv_size), +- ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0); +- return ggml_concat(ctx, kg_v, kc_v, 1); +- } +- +- const uint64_t n_embd_k_gqa = k->ne[0]; + ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * k_s = layer.k_per_stream[s_cache]; ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ ggml_tensor * kg_v = ggml_view_4d(ctx, k_s, ++ n_embd_head_k, n_head_gpu, n_kv, 1, ++ ggml_row_size(k_s->type, n_embd_head_k), ++ ggml_row_size(k_s->type, n_embd_gpu), ++ ggml_row_size(k_s->type, n_embd_gpu*kv_size), ++ 0); ++ ggml_tensor * kc_v = ggml_view_4d(ctx, kc_s, ++ n_embd_head_k, n_head_cpu, n_kv, 1, ++ ggml_row_size(kc_s->type, n_embd_head_k), ++ ggml_row_size(kc_s->type, n_embd_cpu), ++ ggml_row_size(kc_s->type, n_embd_cpu*kv_size), ++ 0); ++ return ggml_concat(ctx, kg_v, kc_v, 1); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++ } ++ ++ // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream axis ++ // (dim 3). In unified mode (sinfo.s1 == sinfo.s0 == 0) the loop does ++ // not fire, yielding a single 4-D view of layer.k_per_stream[0] — ++ // structurally and numerically byte-identical to pre-Phase-4 (same ++ // ne, same nb, same offset 0). In non-unified mode the result is N ++ // contiguous concat ops along dim 3 (the stream axis), one per ++ // covered cache stream. Cost: (ns - 1) concat nodes per layer per turn. ++ auto * k0 = layer.k_per_stream[sinfo.s0]; ++ const uint64_t n_embd_k_gqa = k0->ne[0]; + assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il)); + +- return ggml_view_4d(ctx, k, +- hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns, +- ggml_row_size(k->type, hparams.n_embd_head_k(il)), +- ggml_row_size(k->type, n_embd_k_gqa), +- ggml_row_size(k->type, n_embd_k_gqa*kv_size), +- ggml_row_size(k->type, n_embd_k_gqa*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * k_s = layer.k_per_stream[s_cache]; ++ return ggml_view_4d(ctx, k_s, ++ hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, 1, ++ ggml_row_size(k_s->type, hparams.n_embd_head_k(il)), ++ ggml_row_size(k_s->type, n_embd_k_gqa), ++ ggml_row_size(k_s->type, n_embd_k_gqa*kv_size), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; + } + + ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + + const auto & layer = layers[ikv]; +- auto * v = layer.v; + + const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; + +- // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The split +- // is gated off when v_trans at construction, so this branch is always the +- // non-transposed layout (heads in dim 1, same as get_k). +- if (layer.gpu_heads > 0 && layer.v_cpu) { ++ // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The ++ // split is gated off when v_trans at construction, so this branch is ++ // always the non-transposed layout (heads in dim 1, same as get_k). ++ // ++ // opencoti F4 M3 Phase 4-D: stream-aware per-stream concat — same ++ // pattern as get_k's M2 branch. ++ if (layer.gpu_heads > 0 && !layer.v_cpu_per_stream.empty() && layer.v_cpu_per_stream[0]) { + const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); + const uint32_t n_head_gpu = layer.gpu_heads; + const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; + const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_v; + const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_v; +- auto * vc = layer.v_cpu; +- ggml_tensor * vg_v = ggml_view_4d(ctx, v, +- n_embd_head_v, n_head_gpu, n_kv, ns, +- ggml_row_size(v->type, n_embd_head_v), +- ggml_row_size(v->type, n_embd_gpu), +- ggml_row_size(v->type, n_embd_gpu*kv_size), +- ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0); +- ggml_tensor * vc_v = ggml_view_4d(ctx, vc, +- n_embd_head_v, n_head_cpu, n_kv, ns, +- ggml_row_size(vc->type, n_embd_head_v), +- ggml_row_size(vc->type, n_embd_cpu), +- ggml_row_size(vc->type, n_embd_cpu*kv_size), +- ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0); +- return ggml_concat(ctx, vg_v, vc_v, 1); +- } +- +- const uint64_t n_embd_v_gqa = v->ne[0]; ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache]; ++ ggml_tensor * vg_v = ggml_view_4d(ctx, v_s, ++ n_embd_head_v, n_head_gpu, n_kv, 1, ++ ggml_row_size(v_s->type, n_embd_head_v), ++ ggml_row_size(v_s->type, n_embd_gpu), ++ ggml_row_size(v_s->type, n_embd_gpu*kv_size), ++ 0); ++ ggml_tensor * vc_v = ggml_view_4d(ctx, vc_s, ++ n_embd_head_v, n_head_cpu, n_kv, 1, ++ ggml_row_size(vc_s->type, n_embd_head_v), ++ ggml_row_size(vc_s->type, n_embd_cpu), ++ ggml_row_size(vc_s->type, n_embd_cpu*kv_size), ++ 0); ++ return ggml_concat(ctx, vg_v, vc_v, 1); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++ } ++ ++ // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream ++ // axis. Same shape as get_k, but with v_trans branching. Unified mode ++ // (sinfo.s1 == sinfo.s0 == 0) keeps a single 4-D view → byte-identical. ++ auto * v0 = layer.v_per_stream[sinfo.s0]; ++ const uint64_t n_embd_v_gqa = v0->ne[0]; + + // [TAG_V_CACHE_VARIABLE] + assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il)); + + if (!v_trans) { + // note: v->nb[1] <= v->nb[2] +- return ggml_view_4d(ctx, v, +- hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, ns, +- ggml_row_size(v->type, hparams.n_embd_head_v(il)), // v->nb[1] +- ggml_row_size(v->type, n_embd_v_gqa), // v->nb[2] +- ggml_row_size(v->type, n_embd_v_gqa*kv_size), // v->nb[3] +- ggml_row_size(v->type, n_embd_v_gqa*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ return ggml_view_4d(ctx, v_s, ++ hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, 1, ++ ggml_row_size(v_s->type, hparams.n_embd_head_v(il)), ++ ggml_row_size(v_s->type, n_embd_v_gqa), ++ ggml_row_size(v_s->type, n_embd_v_gqa*kv_size), ++ 0); ++ }; ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++ } ++ ++ // v_trans branch: note v->nb[1] > v->nb[2] ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ return ggml_view_4d(ctx, v_s, ++ n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), 1, ++ ggml_row_size(v_s->type, kv_size*hparams.n_embd_head_v(il)), ++ ggml_row_size(v_s->type, kv_size), ++ ggml_row_size(v_s->type, kv_size*n_embd_v_gqa), ++ 0); ++ }; ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); + } +- +- // note: v->nb[1] > v->nb[2] +- return ggml_view_4d(ctx, v, +- n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), ns, +- ggml_row_size(v->type, kv_size*hparams.n_embd_head_v(il)), // v->nb[1] +- ggml_row_size(v->type, kv_size), // v->nb[2] +- ggml_row_size(v->type, kv_size*n_embd_v_gqa), // v->nb[3] +- ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0); ++ return result; + } + + // opencoti F5 M3 neo — head-split-aware accessors. These factor the per-half +@@ -1824,7 +2027,13 @@ bool llama_kv_cache::headinfer_split_active(int32_t il) const { + return false; + } + const auto & layer = layers[it->second]; +- return layer.gpu_heads > 0 && layer.k_cpu != nullptr; ++ // opencoti F4 M3 Phase 4: per-stream k_cpu vector is empty when no split, ++ // populated when gpu_heads > 0. Stream 0 always exists in unified mode ++ // (the only mode where split engages today; the M2 construction gate ++ // requires n_stream == 1). ++ return layer.gpu_heads > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0] != nullptr; + } + + uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const { +@@ -1840,93 +2049,131 @@ ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split"); + +- auto * k = layer.k; +- const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ // opencoti F4 M3 Phase 4-D: per-stream M2 GPU subset. In unified ++ // mode (sinfo.s1 == sinfo.s0 == 0) the loop runs once and yields ++ // a single 4-D view of layer.k_per_stream[0]'s GPU half — ++ // byte-identical to pre-Phase-4 (same ne, same nb, offset 0). ++ const uint64_t kv_size = get_size(); + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); + const uint32_t n_head_gpu = layer.gpu_heads; + const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_k; + +- return ggml_view_4d(ctx, k, +- n_embd_head_k, n_head_gpu, n_kv, ns, +- ggml_row_size(k->type, n_embd_head_k), +- ggml_row_size(k->type, n_embd_gpu), +- ggml_row_size(k->type, n_embd_gpu*kv_size), +- ggml_row_size(k->type, n_embd_gpu*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * k_s = layer.k_per_stream[s_cache]; ++ return ggml_view_4d(ctx, k_s, ++ n_embd_head_k, n_head_gpu, n_kv, 1, ++ ggml_row_size(k_s->type, n_embd_head_k), ++ ggml_row_size(k_s->type, n_embd_gpu), ++ ggml_row_size(k_s->type, n_embd_gpu*kv_size), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; + } + + ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + const auto & layer = layers[ikv]; +- GGML_ASSERT(layer.gpu_heads > 0 && layer.k_cpu && "get_k_cpu called without an active headinfer split"); +- +- auto * kc = layer.k_cpu; +- const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ GGML_ASSERT(layer.gpu_heads > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0] ++ && "get_k_cpu called without an active headinfer split"); ++ ++ // opencoti F4 M3 Phase 4-D: per-stream M2 CPU subset. Same pattern ++ // as get_k_gpu; unified mode collapses to a single 4-D view. ++ const uint64_t kv_size = get_size(); + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); + const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; + const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_k; + +- return ggml_view_4d(ctx, kc, +- n_embd_head_k, n_head_cpu, n_kv, ns, +- ggml_row_size(kc->type, n_embd_head_k), +- ggml_row_size(kc->type, n_embd_cpu), +- ggml_row_size(kc->type, n_embd_cpu*kv_size), +- ggml_row_size(kc->type, n_embd_cpu*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ return ggml_view_4d(ctx, kc_s, ++ n_embd_head_k, n_head_cpu, n_kv, 1, ++ ggml_row_size(kc_s->type, n_embd_head_k), ++ ggml_row_size(kc_s->type, n_embd_cpu), ++ ggml_row_size(kc_s->type, n_embd_cpu*kv_size), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; + } + + ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split"); +- // M2's gate is `!v_trans && n_stream == 1` — when split is active the +- // V layout is always non-transposed (matches get_v's non-split branch +- // shape modulo head count). Same head-dim-1 stride pattern as K. ++ // M2 requires non-transposed V (FA on). The split-active branch in ++ // get_v/cpy_v also requires this. Per-stream view + concat on stream ++ // axis. Phase 4-D: lifted the n_stream == 1 gate. + GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V"); + +- auto * v = layer.v; +- const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ const uint64_t kv_size = get_size(); + const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); + const uint32_t n_head_gpu = layer.gpu_heads; + const uint64_t n_embd_gpu = (uint64_t) n_head_gpu * n_embd_head_v; + +- return ggml_view_4d(ctx, v, +- n_embd_head_v, n_head_gpu, n_kv, ns, +- ggml_row_size(v->type, n_embd_head_v), +- ggml_row_size(v->type, n_embd_gpu), +- ggml_row_size(v->type, n_embd_gpu*kv_size), +- ggml_row_size(v->type, n_embd_gpu*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ return ggml_view_4d(ctx, v_s, ++ n_embd_head_v, n_head_gpu, n_kv, 1, ++ ggml_row_size(v_s->type, n_embd_head_v), ++ ggml_row_size(v_s->type, n_embd_gpu), ++ ggml_row_size(v_s->type, n_embd_gpu*kv_size), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; + } + + ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + const auto & layer = layers[ikv]; +- GGML_ASSERT(layer.gpu_heads > 0 && layer.v_cpu && "get_v_cpu called without an active headinfer split"); ++ GGML_ASSERT(layer.gpu_heads > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[0] ++ && "get_v_cpu called without an active headinfer split"); + GGML_ASSERT(!v_trans && "headinfer split requires non-transposed V"); + +- auto * vc = layer.v_cpu; +- const uint64_t kv_size = get_size(); +- const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; ++ // opencoti F4 M3 Phase 4-D: per-stream M2 CPU V subset. ++ const uint64_t kv_size = get_size(); + const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); + const uint32_t n_head_cpu = hparams.n_head_kv(il) - layer.gpu_heads; + const uint64_t n_embd_cpu = (uint64_t) n_head_cpu * n_embd_head_v; + +- return ggml_view_4d(ctx, vc, +- n_embd_head_v, n_head_cpu, n_kv, ns, +- ggml_row_size(vc->type, n_embd_head_v), +- ggml_row_size(vc->type, n_embd_cpu), +- ggml_row_size(vc->type, n_embd_cpu*kv_size), +- ggml_row_size(vc->type, n_embd_cpu*kv_size)*sinfo.s0); ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache]; ++ return ggml_view_4d(ctx, vc_s, ++ n_embd_head_v, n_head_cpu, n_kv, 1, ++ ggml_row_size(vc_s->type, n_embd_head_v), ++ ggml_row_size(vc_s->type, n_embd_cpu), ++ ggml_row_size(vc_s->type, n_embd_cpu*kv_size), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; + } + + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { +- GGML_UNUSED(sinfo); +- + const int32_t ikv = map_layer_ids.at(il); + + const auto & layer = layers[ikv]; +- ggml_tensor * k = layer.k; + + const int64_t n_embd_head = k_cur->ne[0]; + const int64_t n_head = k_cur->ne[1]; +@@ -1941,46 +2188,116 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + // opencoti F5 M2 headinfer — scatter the incoming heads into the GPU and + // host subsets. The first gpu_heads heads (contiguous in k_cur's dim 0..1) + // store into k; the rest store into k_cpu, both at the same cell indices +- // (k_idxs). The single returned dep node ties both set_rows into the +- // caller's one ggml_build_forward_expand (its value is never read). Split +- // only happens for n_stream==1 (gated at construction) — no per-stream +- // reshape needed. +- if (layer.gpu_heads > 0 && layer.k_cpu) { ++ // (k_idxs). The single returned dep node ties all set_rows into the ++ // caller's one ggml_build_forward_expand (its value is never read). ++ // ++ // opencoti F4 M3 Phase 4-D: per-stream M2 scatter — for each batch ++ // stream s, slice k_cur and k_idxs by stream, split the head-axis ++ // slice into GPU + CPU halves, and emit 2 set_rows per stream against ++ // the per-stream tensors. In unified mode (sinfo.n_stream() == 1) this ++ // collapses to exactly 2 set_rows on stream 0's tensors — byte-identical ++ // to pre-Phase-4. ++ if (layer.gpu_heads > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0]) { + const int64_t g = layer.gpu_heads; + const int64_t c = n_head - g; +- ggml_tensor * kc = layer.k_cpu; +- ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur, n_embd_head*g, n_tokens, k_cur->nb[2], 0); +- ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur, n_embd_head*c, n_tokens, k_cur->nb[2], (size_t) (g*k_cur->nb[1])); +- ggml_tensor * store_g = ggml_set_rows(ctx, k, cur_g, k_idxs); +- ggml_tensor * store_c = ggml_set_rows(ctx, kc, cur_c, k_idxs); +- return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0)); ++ const uint32_t batch_ns = sinfo.n_stream(); ++ GGML_ASSERT(batch_ns >= 1); ++ ++ const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[s]]; ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[sinfo.strm[s]]; ++ ++ // k_cur is [head_dim, n_head, n_tokens]; nb[2] is the stride ++ // between tokens. The per-stream slice is the contiguous block ++ // of tokens_per_stream tokens at offset s*tokens_per_stream. ++ const size_t per_stream_off = (size_t) s * tokens_per_stream * k_cur->nb[2]; ++ ggml_tensor * cur_g = ggml_view_2d(ctx, k_cur, ++ n_embd_head*g, tokens_per_stream, ++ k_cur->nb[2], per_stream_off); ++ // CPU half: remaining c heads, offset by g heads (g rows in the head-merged dim 0) ++ ggml_tensor * cur_c = ggml_view_2d(ctx, k_cur, ++ n_embd_head*c, tokens_per_stream, ++ k_cur->nb[2], per_stream_off + (size_t)(g*k_cur->nb[1])); ++ ggml_tensor * idxs_s = (batch_ns == 1) ++ ? k_idxs ++ : ggml_view_1d(ctx, k_idxs, tokens_per_stream, ++ (size_t) s * tokens_per_stream * k_idxs->nb[0]); ++ ggml_tensor * store_g = ggml_set_rows(ctx, k_s, cur_g, idxs_s); ++ ggml_tensor * store_c = ggml_set_rows(ctx, kc_s, cur_c, idxs_s); ++ ggml_tensor * pair = ggml_add(ctx, ++ ggml_view_1d(ctx, store_g, 1, 0), ++ ggml_view_1d(ctx, store_c, 1, 0)); ++ if (!tie) { ++ tie = pair; ++ } else { ++ tie = ggml_add(ctx, ++ ggml_view_1d(ctx, tie, 1, 0), ++ ggml_view_1d(ctx, pair, 1, 0)); ++ } ++ } ++ return tie; + } + + k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0); + +- const int64_t n_stream = k->ne[2]; +- +- if (n_stream > 1) { +- const int64_t kv_size = get_size(); +- +- assert(n_embd_gqa == k->ne[0]); +- assert(kv_size == k->ne[1]); +- +- // merge the buffer across all streams because the idxs are global +- k = ggml_reshape_2d(ctx, k, n_embd_gqa, kv_size*n_stream); ++ // opencoti F4 M3 Phase 4-C: per-stream scatter via N separate ++ // ggml_set_rows ops (option C-2). In unified mode (sinfo.n_stream() ++ // == 1) the fast path collapses to one ggml_set_rows against ++ // layer.k_per_stream[sinfo.strm[0]] — byte-identical to pre-Phase-4. ++ // In non-unified mode (sinfo.n_stream() > 1), set_input_k_idxs has ++ // emitted LOCAL indices (no global offset), so each per-stream slice ++ // of k_idxs is valid against its own k_per_stream[cache_strm] tensor. ++ // The N stores are tied into a single dep node via ggml_add so the ++ // caller's one ggml_build_forward_expand picks them all up. ++ // ++ // C-1 (concat-then-scatter) is not expressible with per-stream owning ++ // tensors in ggml: ggml_concat materializes a new buffer rather than ++ // creating a view into its inputs, so writes routed through the concat ++ // result would not land in the per-stream tensors. C-2 is the natural ++ // shape that preserves per-stream buffer ownership. ++ const uint32_t batch_ns = sinfo.n_stream(); ++ GGML_ASSERT(batch_ns >= 1); ++ ++ if (batch_ns == 1) { ++ ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[0]]; ++ return ggml_set_rows(ctx, k_s, k_cur, k_idxs); ++ } ++ ++ const int64_t tokens_per_stream = n_tokens / (int64_t) batch_ns; ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[s]]; ++ ggml_tensor * cur_s = ggml_view_2d(ctx, k_cur, ++ n_embd_gqa, tokens_per_stream, ++ k_cur->nb[1], ++ (size_t) s * tokens_per_stream * k_cur->nb[1]); ++ ggml_tensor * idxs_s = ggml_view_1d(ctx, k_idxs, ++ tokens_per_stream, ++ (size_t) s * tokens_per_stream * k_idxs->nb[0]); ++ ggml_tensor * store_s = ggml_set_rows(ctx, k_s, cur_s, idxs_s); ++ if (!tie) { ++ tie = store_s; ++ } else { ++ tie = ggml_add(ctx, ++ ggml_view_1d(ctx, tie, 1, 0), ++ ggml_view_1d(ctx, store_s, 1, 0)); ++ } + } +- +- // store the current K values into the cache +- return ggml_set_rows(ctx, k, k_cur, k_idxs); ++ return tie; + } + + ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const { +- GGML_UNUSED(sinfo); +- + const int32_t ikv = map_layer_ids.at(il); + + const auto & layer = layers[ikv]; +- auto * v = layer.v; + + const int64_t n_embd_head = v_cur->ne[0]; + const int64_t n_head = v_cur->ne[1]; +@@ -1992,38 +2309,109 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]); + + // opencoti F5 M2 headinfer — head-scatter into the GPU + host subsets. +- // Reached only when !v_trans and n_stream==1 (the split is gated off +- // otherwise at construction), so this mirrors the cpy_k store exactly. +- if (layer.gpu_heads > 0 && layer.v_cpu) { ++ // Reached only when !v_trans (the split is gated off at construction ++ // otherwise). ++ // ++ // opencoti F4 M3 Phase 4-D: per-stream M2 V scatter — same shape as ++ // cpy_k's M2 branch. Unified mode collapses to 2 set_rows on stream 0, ++ // byte-identical to pre-Phase-4. ++ if (layer.gpu_heads > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[0]) { + const int64_t g = layer.gpu_heads; + const int64_t c = n_head - g; +- ggml_tensor * vc = layer.v_cpu; +- ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur, n_embd_head*g, n_tokens, v_cur->nb[2], 0); +- ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur, n_embd_head*c, n_tokens, v_cur->nb[2], (size_t) (g*v_cur->nb[1])); +- ggml_tensor * store_g = ggml_set_rows(ctx, v, cur_g, v_idxs); +- ggml_tensor * store_c = ggml_set_rows(ctx, vc, cur_c, v_idxs); +- return ggml_add(ctx, ggml_view_1d(ctx, store_g, 1, 0), ggml_view_1d(ctx, store_c, 1, 0)); ++ const uint32_t batch_ns = sinfo.n_stream(); ++ GGML_ASSERT(batch_ns >= 1); ++ ++ const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[s]]; ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[sinfo.strm[s]]; ++ ++ const size_t per_stream_off = (size_t) s * tokens_per_stream * v_cur->nb[2]; ++ ggml_tensor * cur_g = ggml_view_2d(ctx, v_cur, ++ n_embd_head*g, tokens_per_stream, ++ v_cur->nb[2], per_stream_off); ++ ggml_tensor * cur_c = ggml_view_2d(ctx, v_cur, ++ n_embd_head*c, tokens_per_stream, ++ v_cur->nb[2], per_stream_off + (size_t)(g*v_cur->nb[1])); ++ ggml_tensor * idxs_s = (batch_ns == 1) ++ ? v_idxs ++ : ggml_view_1d(ctx, v_idxs, tokens_per_stream, ++ (size_t) s * tokens_per_stream * v_idxs->nb[0]); ++ ggml_tensor * store_g = ggml_set_rows(ctx, v_s, cur_g, idxs_s); ++ ggml_tensor * store_c = ggml_set_rows(ctx, vc_s, cur_c, idxs_s); ++ ggml_tensor * pair = ggml_add(ctx, ++ ggml_view_1d(ctx, store_g, 1, 0), ++ ggml_view_1d(ctx, store_c, 1, 0)); ++ if (!tie) { ++ tie = pair; ++ } else { ++ tie = ggml_add(ctx, ++ ggml_view_1d(ctx, tie, 1, 0), ++ ggml_view_1d(ctx, pair, 1, 0)); ++ } ++ } ++ return tie; + } + +- const int64_t n_stream = v->ne[2]; ++ // opencoti F4 M3 Phase 4-C: per-stream V scatter. Two sub-branches ++ // depending on v_trans (FA vs non-FA layout). ++ // ++ // !v_trans (FA on): same shape as cpy_k — N separate ggml_set_rows on ++ // the 2-D-merged v_cur, sliced per stream. set_input_v_idxs emits ++ // LOCAL per-stream indices in non-unified mode. Unified mode (batch_ns ++ // == 1) keeps the simple one-set_rows fast path, byte-identical. ++ // ++ // v_trans (FA off, V transposed): each row is a single head element. ++ // The unified-mode path is preserved; multi-stream + v_trans is not a ++ // current workload (the v_trans layout requires no FA, while parallel ++ // multi-stream batches are always paired with FA). Asserted out as a ++ // safety net; future Phase 4-D/E may lift if a workload appears. ++ const uint32_t batch_ns = sinfo.n_stream(); ++ GGML_ASSERT(batch_ns >= 1); + +- // take this branch when FA is enabled (the V cache is not transposed) + if (!v_trans) { + v_cur = ggml_view_2d(ctx, v_cur, n_embd_gqa, n_tokens, v_cur->nb[2], 0); + +- if (n_stream > 1) { +- const int64_t kv_size = get_size(); +- +- assert(n_embd_gqa == v->ne[0]); +- assert(kv_size == v->ne[1]); +- +- // merge the buffer across all streams because the idxs are global +- v = ggml_reshape_2d(ctx, v, n_embd_gqa, kv_size*n_stream); ++ if (batch_ns == 1) { ++ ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[0]]; ++ return ggml_set_rows(ctx, v_s, v_cur, v_idxs); ++ } ++ ++ const int64_t tokens_per_stream = n_tokens / (int64_t) batch_ns; ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[s]]; ++ ggml_tensor * cur_s = ggml_view_2d(ctx, v_cur, ++ n_embd_gqa, tokens_per_stream, ++ v_cur->nb[1], ++ (size_t) s * tokens_per_stream * v_cur->nb[1]); ++ ggml_tensor * idxs_s = ggml_view_1d(ctx, v_idxs, ++ tokens_per_stream, ++ (size_t) s * tokens_per_stream * v_idxs->nb[0]); ++ ggml_tensor * store_s = ggml_set_rows(ctx, v_s, cur_s, idxs_s); ++ if (!tie) { ++ tie = store_s; ++ } else { ++ tie = ggml_add(ctx, ++ ggml_view_1d(ctx, tie, 1, 0), ++ ggml_view_1d(ctx, store_s, 1, 0)); ++ } + } +- +- return ggml_set_rows(ctx, v, v_cur, v_idxs); ++ return tie; + } + ++ // v_trans path: not a multi-stream workload; preserve unified-mode ++ // behavior exactly and assert multi-stream out. ++ GGML_ASSERT(batch_ns == 1 && "v_trans + multi-stream is unsupported; v_trans implies !FA which is mutually exclusive with parallel batches"); ++ ggml_tensor * v = layer.v_per_stream[sinfo.strm[0]]; ++ + if (ggml_row_size(v_cur->type, n_embd_gqa) == v_cur->nb[2]) { + // we can merge dims 0, 1 and 2 + v_cur = ggml_reshape_2d(ctx, v_cur, n_embd_gqa, n_tokens); +@@ -2119,11 +2507,23 @@ void llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ub + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + int64_t * data = (int64_t *) dst->data; + +- for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { +- const int64_t offs = sinfo.strm[s]*get_size(); ++ // opencoti F4 M3 Phase 4-C/F: emit LOCAL per-stream indices unconditionally. ++ // Phase 4 ALWAYS scatters to per-stream tensors (cpy_k routes to ++ // k_per_stream[sinfo.strm[s]] which is sized kv_size, not kv_size*n_stream), ++ // so the index must be local to that stream's tensor. In unified mode ++ // (n_stream == 1, sinfo.strm[0] == 0) pre-Phase-4 offs was 0 already, ++ // so this is byte-identical. ++ // ++ // Phase 4-F bug fix: the previous version conditioned the offset ++ // on sinfo.n_stream() (per-batch streams), which left it global ++ // when a --parallel N server served a single-session batch ++ // (sinfo.n_stream() == 1 but sinfo.strm[0] != 0). The resulting ++ // global indices wrote out-of-bounds against the per-stream ++ // tensor and crashed CUDA decode. + ++ for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + for (uint32_t i = 0; i < sinfo.size(); ++i) { +- data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i]; ++ data[s*sinfo.size() + i] = sinfo.idxs[s][i]; + } + } + } +@@ -2135,12 +2535,15 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + int64_t * data = (int64_t *) dst->data; + ++ // opencoti F4 M3 Phase 4-C/F: same LOCAL-only fix as set_input_k_idxs. ++ // The v_trans branch's offset is also dropped — v_trans + multi-stream ++ // is asserted out in cpy_v, but the unified-mode v_trans path here is ++ // byte-identical regardless (sinfo.strm[0] == 0 makes offs == 0). ++ + if (!v_trans) { + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { +- const int64_t offs = sinfo.strm[s]*get_size(); +- + for (uint32_t i = 0; i < sinfo.size(); ++i) { +- data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i]; ++ data[s*sinfo.size() + i] = sinfo.idxs[s][i]; + } + } + } else { +@@ -2150,11 +2553,9 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub + const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa_max(); + + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { +- const int64_t offs = sinfo.strm[s]*kv_size*n_embd_v_gqa; +- + for (uint32_t i = 0; i < sinfo.size(); ++i) { + for (uint32_t j = 0; j < n_embd_v_gqa; ++j) { +- data[s*sinfo.size()*n_embd_v_gqa + i*n_embd_v_gqa + j] = offs + j*kv_size + sinfo.idxs[s][i]; ++ data[s*sinfo.size()*n_embd_v_gqa + i*n_embd_v_gqa + j] = j*kv_size + sinfo.idxs[s][i]; + } + } + } +@@ -2459,10 +2860,16 @@ size_t llama_kv_cache::total_size() const { + } + + size_t llama_kv_cache::size_k_bytes() const { ++ // opencoti F4 M3 Phase 4: sum bytes across all per-stream tensors of ++ // each layer. In unified mode (n_stream == 1) this is byte-identical ++ // to ggml_nbytes(layer.k) of pre-Phase-4. M2's host subset is ++ // accounted separately (it lives in a different buffer-type). + size_t size_k_bytes = 0; + + for (const auto & layer : layers) { +- size_k_bytes += ggml_nbytes(layer.k); ++ for (auto * k : layer.k_per_stream) { ++ if (k) size_k_bytes += ggml_nbytes(k); ++ } + } + + return size_k_bytes; +@@ -2472,7 +2879,9 @@ size_t llama_kv_cache::size_v_bytes() const { + size_t size_v_bytes = 0; + + for (const auto & layer : layers) { +- size_v_bytes += layer.v ? ggml_nbytes(layer.v) : 0; ++ for (auto * v : layer.v_per_stream) { ++ if (v) size_v_bytes += ggml_nbytes(v); ++ } + } + + return size_v_bytes; +@@ -2570,6 +2979,19 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co + + const auto & cparams = lctx->get_cparams(); + ++ // opencoti F4 M3 Phase 4-C: k-shift loops over per-stream tensors. ++ // With per-stream owning tensors, each stream's K is a separate buffer, ++ // so we build N rope-shift ops per layer (one per stream). The k_shift ++ // input tensor has length get_size()*n_stream, with stream s occupying ++ // indices [s*get_size(), (s+1)*get_size()) — set_input_k_shift writes ++ // them in that order. Each per-stream rope-shift consumes its ++ // contiguous slice via ggml_view_1d on k_shift. ++ // ++ // In unified mode (n_stream == 1), the loop runs once with stream 0, ++ // and the k_shift view degenerates to the full tensor (offset 0, ++ // length get_size()). Byte-identical to pre-Phase-4 — same rope-shift ++ // op shape and inputs. ++ + for (const auto & layer : layers) { + const uint32_t il = layer.il; + +@@ -2585,16 +3007,24 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co + + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + +- ggml_tensor * k = +- ggml_view_3d(ctx, layer.k, +- n_rot, n_head_kv, get_size()*n_stream, +- ggml_row_size(layer.k->type, n_embd_head_k), +- ggml_row_size(layer.k->type, n_embd_k_gqa), +- ggml_row_size(layer.k->type, n_embd_nope)); ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ ggml_tensor * layer_k = layer.k_per_stream[s]; ++ ggml_tensor * k = ++ ggml_view_3d(ctx, layer_k, ++ n_rot, n_head_kv, get_size(), ++ ggml_row_size(layer_k->type, n_embd_head_k), ++ ggml_row_size(layer_k->type, n_embd_k_gqa), ++ ggml_row_size(layer_k->type, n_embd_nope)); ++ ++ ggml_tensor * k_shift_s = (n_stream == 1) ++ ? inp->k_shift ++ : ggml_view_1d(ctx, inp->k_shift, get_size(), ++ (size_t) s * get_size() * inp->k_shift->nb[0]); + +- ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); ++ ggml_tensor * cur = build_rope_shift(cparams, ctx, k, k_shift_s, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); + +- ggml_build_forward_expand(gf, cur); ++ ggml_build_forward_expand(gf, cur); ++ } + } + + res->add_input(std::move(inp)); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -286,19 +286,57 @@ private: + // note: can be different from the layer index in the KV cache + uint32_t il; + +- ggml_tensor * k; +- ggml_tensor * v; +- ++ // opencoti F4 M3 Phase 4 — per-stream tensor split. ++ // Each element is an OWNING 3-D tensor of shape ++ // [n_embd_k_gqa, kv_size, 1] (dim 3 == 1 keeps stride math ++ // identical to the pre-Phase-4 single-tensor layout). Vector ++ // size always equals n_stream. ++ // ++ // Unified mode (n_stream == 1): k_per_stream[0] is byte- ++ // identical to the pre-Phase-4 `k` field (same buft, same ++ // ctx, same offsets). Non-unified mode (n_stream > 1): each ++ // stream's tensor lives in its own (possibly per-stream) ++ // backend buffer (Phase 4-B); cross-stream operations are ++ // buffer-to-buffer copies. ++ std::vector k_per_stream; ++ std::vector v_per_stream; ++ ++ // k_stream[s] / v_stream[s] are 2-D views into ++ // k_per_stream[s] / v_per_stream[s]. Kept for backward compat ++ // with consumers that index by stream; same shape and offset ++ // semantics as before. With per-stream tensors the view's ++ // offset within its owning tensor is always 0 (each stream ++ // owns the whole 3-D extent of its tensor with dim 3 == 1). + std::vector k_stream; + std::vector v_stream; + +- // opencoti F5 M2 headinfer — head-residency split. When gpu_heads > 0, +- // `k`/`v` hold the first gpu_heads KV heads on the layer's (GPU) buffer +- // and `k_cpu`/`v_cpu` hold the remaining (n_head_kv - gpu_heads) heads +- // on a host buffer. gpu_heads == 0 means no split (k/v hold all heads). +- ggml_tensor * k_cpu = nullptr; +- ggml_tensor * v_cpu = nullptr; ++ // opencoti F5 M2 headinfer — per-stream now. When gpu_heads > 0, ++ // k_per_stream[s] / v_per_stream[s] hold the first gpu_heads ++ // KV heads on the layer's device buffer, and ++ // k_cpu_per_stream[s] / v_cpu_per_stream[s] hold the remaining ++ // (n_head_kv - gpu_heads) heads on the layer's host buffer. ++ // Vectors are empty when no split (gpu_heads == 0). ++ std::vector k_cpu_per_stream; ++ std::vector v_cpu_per_stream; + uint32_t gpu_heads = 0; ++ ++ // Convenience accessors for unified-mode (n_stream == 1) call ++ // sites. Returns nullptr in non-unified mode — that's the ++ // signal that the caller must use per-stream access instead. ++ // Phase 4-A uses these at sites that haven't yet been migrated ++ // to per-stream awareness; Phase 4-C/D lift them. ++ inline ggml_tensor * k_unified() const { ++ return k_per_stream.size() == 1 ? k_per_stream[0] : nullptr; ++ } ++ inline ggml_tensor * v_unified() const { ++ return v_per_stream.size() == 1 ? v_per_stream[0] : nullptr; ++ } ++ inline ggml_tensor * k_cpu_unified() const { ++ return k_cpu_per_stream.size() == 1 ? k_cpu_per_stream[0] : nullptr; ++ } ++ inline ggml_tensor * v_cpu_unified() const { ++ return v_cpu_per_stream.size() == 1 ? v_cpu_per_stream[0] : nullptr; ++ } + }; + + bool v_trans = true; // the value tensor is transposed diff --git a/patches/0032-m2-state-io.patch b/patches/0032-m2-state-io.patch new file mode 100644 index 0000000000000000000000000000000000000000..57c67952c83e202413248fd94008d0d5f60a5006 --- /dev/null +++ b/patches/0032-m2-state-io.patch @@ -0,0 +1,205 @@ +From: opencoti +Subject: F5 M2 — M2-aware state I/O (bug-222 fix) + +Bench: perf/llamafile/advanced-kv-stack.bench.ts (Gemma 4 A4B + 31B), bug-222 fix +Milestone: F5 M2 hardening — see docs/features/advanced_kv.md, .wolf/buglog.json bug-222 + +Pre-existing M2 head split bug surfaced on Gemma 4 iswa: state_write_data / +state_read_data read/write the K and V cache assuming a single tensor per +layer with full n_embd_k_gqa / n_embd_v_gqa row size. With M2 head split +active the layer's tensor (k_stream[s] / v_stream[s]) holds ONLY the GPU +half (n_embd_k_gpu width), and the CPU half lives in a separate +k_cpu_per_stream[s] / v_cpu_per_stream[s] tensor. Reading buf_size = +range_size * row_full from a tensor that has range_size * row_gpu bytes +crashes with "tensor read out of bounds" in ggml_backend_tensor_get +(ggml-backend.cpp:314) as soon as the per-range write exceeds the GPU +half's nbytes — reproducible on Gemma 4 A4B & 31B at frac=0.5 with any +prompt that crosses a 512-token microbatch boundary and triggers a +context checkpoint (the cache_prompt server feature snapshots ≥ 634 +cells after the second prompt batch). + +Fix: detect M2 split per layer (layer.gpu_heads > 0 && +layer.k_cpu_per_stream[s]), emit GPU half (row_gpu) then CPU half +(row_cpu) per range. Total bytes per range stay at range_size * row_full +(header advertises the full row size unchanged) so io stream accounting +is preserved; on-disk byte order within a range differs from upstream +(GPU-first|CPU-first vs interleaved) but writer/reader are symmetric so +round-trips are consistent. M2 inactive (no head_cpu) path is taken +byte-identical to upstream — preserves Qwen / unified-mode behavior. +V branch only fires in !v_trans (M2 is gated off when v_trans). + +Validated: + - Gemma 4 A4B-98e-v6-coder-it-Q4_K_M: M5 stack bench 8/9 PASS + (C5 byte-equality A2/A3 PASS, C9 perf threshold known cost). + - Gemma 4 31B-it-Q4_K_M: M5 stack bench 8/9 PASS, same shape. + - Qwen 2.5 1.5B Instruct: regression shield — M5 8/9, M2 6/6, M3 4/4, + M0 5/5 all PASS unchanged. + +Bug-222 reproducer: /tmp/gemma-crash-probe.py (cycles reps 10..50 of the +bench prompt against an A2 server; pre-fix crashes at reps=30, post-fix +clean through reps=35). + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -3190,11 +3190,43 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t + const uint64_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa); + io.write(&k_size_row, sizeof(k_size_row)); + +- // Read each range of cells of k_size length and write out +- for (const auto & range : cr.data) { +- const size_t range_size = range.second - range.first; +- const size_t buf_size = range_size * k_size_row; +- io.write_tensor(k, range.first * k_size_row, buf_size); ++ // opencoti bug-222 fix — F5 M2 split-aware state write. ++ // When M2 head split is active, `k` (k_stream[s]) holds only the ++ // GPU half of the layer's heads; the CPU half lives in ++ // k_cpu_per_stream[s]. The pre-fix path read `range_size * k_size_row` ++ // bytes from a tensor that only has `range_size * row_gpu` bytes → ++ // out-of-bounds crash on any range_size that pushes buf_size past ++ // the per-stream GPU tensor's nbytes. Fix: detect M2 split, emit ++ // GPU half then CPU half per range. Total bytes per range equals ++ // `range_size * k_size_row` (the header advertises this), so on-disk ++ // accounting matches; the layout differs only in within-range order ++ // (GPU-first|CPU-first vs interleaved). Reader (state_read_data) is ++ // symmetric so a save/load round-trip is consistent. When M2 is OFF ++ // (no head_cpu) the original path is taken byte-identical. ++ const bool m2_split_active = layer.gpu_heads > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[cr.strm]; ++ if (m2_split_active) { ++ auto * k_cpu = layer.k_cpu_per_stream[cr.strm]; ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint32_t n_embd_k_gpu = layer.gpu_heads * n_embd_head_k; ++ const uint32_t n_embd_k_cpu = (n_head_kv - layer.gpu_heads) * n_embd_head_k; ++ const uint64_t k_row_gpu = ggml_row_size(k->type, n_embd_k_gpu); ++ const uint64_t k_row_cpu = ggml_row_size(k_cpu->type, n_embd_k_cpu); ++ GGML_ASSERT(k_row_gpu + k_row_cpu == k_size_row); ++ for (const auto & range : cr.data) { ++ const size_t range_size = range.second - range.first; ++ io.write_tensor(k, range.first * k_row_gpu, range_size * k_row_gpu); ++ io.write_tensor(k_cpu, range.first * k_row_cpu, range_size * k_row_cpu); ++ } ++ } else { ++ // Read each range of cells of k_size length and write out ++ for (const auto & range : cr.data) { ++ const size_t range_size = range.second - range.first; ++ const size_t buf_size = range_size * k_size_row; ++ io.write_tensor(k, range.first * k_size_row, buf_size); ++ } + } + } + +@@ -3217,11 +3249,33 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t + const uint64_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa); + io.write(&v_size_row, sizeof(v_size_row)); + +- // Read each range of cells of v_size length and write out +- for (const auto & range : cr.data) { +- const size_t range_size = range.second - range.first; +- const size_t buf_size = range_size * v_size_row; +- io.write_tensor(v, range.first * v_size_row, buf_size); ++ // opencoti bug-222 fix — F5 M2 split-aware V state write ++ // (same pattern as K above). M2 is gated off when v_trans, so ++ // this branch is reached only with the non-transposed V layout. ++ const bool m2_split_active = layer.gpu_heads > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[cr.strm]; ++ if (m2_split_active) { ++ auto * v_cpu = layer.v_cpu_per_stream[cr.strm]; ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint32_t n_embd_v_gpu = layer.gpu_heads * n_embd_head_v; ++ const uint32_t n_embd_v_cpu = (n_head_kv - layer.gpu_heads) * n_embd_head_v; ++ const uint64_t v_row_gpu = ggml_row_size(v->type, n_embd_v_gpu); ++ const uint64_t v_row_cpu = ggml_row_size(v_cpu->type, n_embd_v_cpu); ++ GGML_ASSERT(v_row_gpu + v_row_cpu == v_size_row); ++ for (const auto & range : cr.data) { ++ const size_t range_size = range.second - range.first; ++ io.write_tensor(v, range.first * v_row_gpu, range_size * v_row_gpu); ++ io.write_tensor(v_cpu, range.first * v_row_cpu, range_size * v_row_cpu); ++ } ++ } else { ++ // Read each range of cells of v_size length and write out ++ for (const auto & range : cr.data) { ++ const size_t range_size = range.second - range.first; ++ const size_t buf_size = range_size * v_size_row; ++ io.write_tensor(v, range.first * v_size_row, buf_size); ++ } + } + } + } else { +@@ -3433,7 +3487,36 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32 + } + + if (cell_count) { +- if (sinfo.is_contiguous()) { ++ // opencoti bug-222 fix — F5 M2 split-aware state read (K). ++ // Symmetric to state_write_data: when M2 head split is active, ++ // read the GPU half then the CPU half and dispatch to the ++ // respective per-stream tensors. Total bytes equal ++ // `cell_count * k_size_row` (the header advertises this) so the ++ // sequential io stream matches what state_write_data wrote. ++ const bool m2_split_active = layer.gpu_heads > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[strm]; ++ if (m2_split_active) { ++ auto * k_cpu = layer.k_cpu_per_stream[strm]; ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint32_t n_embd_k_gpu = layer.gpu_heads * n_embd_head_k; ++ const uint32_t n_embd_k_cpu = (n_head_kv - layer.gpu_heads) * n_embd_head_k; ++ const uint64_t k_row_gpu = ggml_row_size(k->type, n_embd_k_gpu); ++ const uint64_t k_row_cpu = ggml_row_size(k_cpu->type, n_embd_k_cpu); ++ GGML_ASSERT(k_row_gpu + k_row_cpu == k_size_row); ++ if (sinfo.is_contiguous()) { ++ io.read_tensor(k, sinfo.head() * k_row_gpu, cell_count * k_row_gpu); ++ io.read_tensor(k_cpu, sinfo.head() * k_row_cpu, cell_count * k_row_cpu); ++ } else { ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ io.read_tensor(k, sinfo.idxs[0][i] * k_row_gpu, k_row_gpu); ++ } ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ io.read_tensor(k_cpu, sinfo.idxs[0][i] * k_row_cpu, k_row_cpu); ++ } ++ } ++ } else if (sinfo.is_contiguous()) { + // Fast path: contiguous cells, single memcpy + io.read_tensor(k, sinfo.head() * k_size_row, cell_count * k_size_row); + } else { +@@ -3476,7 +3559,33 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32 + } + + if (cell_count) { +- if (sinfo.is_contiguous()) { ++ // opencoti bug-222 fix — F5 M2 split-aware state read (V), ++ // same pattern as K. M2 is gated off when v_trans, so this ++ // branch is only reached with the non-transposed V layout. ++ const bool m2_split_active = layer.gpu_heads > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[strm]; ++ if (m2_split_active) { ++ auto * v_cpu = layer.v_cpu_per_stream[strm]; ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint32_t n_embd_v_gpu = layer.gpu_heads * n_embd_head_v; ++ const uint32_t n_embd_v_cpu = (n_head_kv - layer.gpu_heads) * n_embd_head_v; ++ const uint64_t v_row_gpu = ggml_row_size(v->type, n_embd_v_gpu); ++ const uint64_t v_row_cpu = ggml_row_size(v_cpu->type, n_embd_v_cpu); ++ GGML_ASSERT(v_row_gpu + v_row_cpu == v_size_row); ++ if (sinfo.is_contiguous()) { ++ io.read_tensor(v, sinfo.head() * v_row_gpu, cell_count * v_row_gpu); ++ io.read_tensor(v_cpu, sinfo.head() * v_row_cpu, cell_count * v_row_cpu); ++ } else { ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ io.read_tensor(v, sinfo.idxs[0][i] * v_row_gpu, v_row_gpu); ++ } ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ io.read_tensor(v_cpu, sinfo.idxs[0][i] * v_row_cpu, v_row_cpu); ++ } ++ } ++ } else if (sinfo.is_contiguous()) { + // Fast path: contiguous cells, single memcpy + io.read_tensor(v, sinfo.head() * v_size_row, cell_count * v_size_row); + } else { diff --git a/patches/0033-cpy-tie-blck-align.patch b/patches/0033-cpy-tie-blck-align.patch new file mode 100644 index 0000000000000000000000000000000000000000..08fe51a3aec1e58ddae53831b0c24226b9444657 --- /dev/null +++ b/patches/0033-cpy-tie-blck-align.patch @@ -0,0 +1,100 @@ +opencoti F5 bug-225 — block-align cpy_k/cpy_v dependency-tie views over quantized KV + +At 256k + q8_0 KV with M2 head split (--headinfer-gpu-heads-frac < 1.0) on Gemma 4 +iswa, server boot crashed during common_init_result's graph_reserve with: + ggml.c:1301: assert(ne % ggml_blck_size(type) == 0) failed + +Root cause: the M2 and Phase-4 branches of cpy_k / cpy_v built a graph-topology +dependency tie via `ggml_view_1d(ctx, store, 1, 0)` over the K/V cache scatter +result. The tie value is never read by computation — it only forces multiple +per-stream / per-head writes to be reachable from one ggml_build_forward_expand +call. At f16 (block size 1), ne=1 passes ggml_row_size silently. At q8_0 +(block size 32), ne=1 fails the assert at graph_reserve before any inference +runs. + +Fix at all 4 callsites (cpy_k M2 branch + Phase-4 multi-stream branch; cpy_v +M2 branch + Phase-4 !v_trans multi-stream branch — 12 view sites total): + - Replace `ne = 1` with `ggml_blck_size(target->type)` so the view passes + the quantized-alignment check. + - Wrap each view in `ggml_cast(..., GGML_TYPE_F32)` so the subsequent + `ggml_add` survives CUDA bin_bcast's F32/F16-only check (binbcast.cu:376). + +At f16 KV both wraps are no-ops (blck_size==1, cast on f16-view is byte-identical +for ne=1 tie semantics never read). At q8_0 the wraps unblock 256k boot. + +Verification: + - Boot at 4096 / 32768 / 262144 + q8_0 + M2 frac=0.5 on Gemma 4 A4B Q4_K_M: + all succeed (no graph_reserve crash). + - RULER vt@4096 + vt@32768 + niah_single_1@{4096,32768} ours-c1 scored 100/100 + against the post-fix binary on 2026-05-29 — confirms no semantic regression + at moderate ctx. + +Related: bug-222 (n_kv/view stride on iswa tail microbatch); bug-224 (concat-F32 +on iswa); bug-226 (256k semantic collapse — different root cause, separate fix). + +Patch order: applies after 0031-per-stream-split (Phase 4) and 0032-m2-state-io. +Cosmocc-only edit; CUDA DSO unchanged. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2231,14 +2231,14 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + ggml_tensor * store_g = ggml_set_rows(ctx, k_s, cur_g, idxs_s); + ggml_tensor * store_c = ggml_set_rows(ctx, kc_s, cur_c, idxs_s); + ggml_tensor * pair = ggml_add(ctx, +- ggml_view_1d(ctx, store_g, 1, 0), +- ggml_view_1d(ctx, store_c, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32)); + if (!tie) { + tie = pair; + } else { + tie = ggml_add(ctx, +- ggml_view_1d(ctx, tie, 1, 0), +- ggml_view_1d(ctx, pair, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32)); + } + } + return tie; +@@ -2287,8 +2287,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + tie = store_s; + } else { + tie = ggml_add(ctx, +- ggml_view_1d(ctx, tie, 1, 0), +- ggml_view_1d(ctx, store_s, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32)); + } + } + return tie; +@@ -2345,14 +2345,14 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + ggml_tensor * store_g = ggml_set_rows(ctx, v_s, cur_g, idxs_s); + ggml_tensor * store_c = ggml_set_rows(ctx, vc_s, cur_c, idxs_s); + ggml_tensor * pair = ggml_add(ctx, +- ggml_view_1d(ctx, store_g, 1, 0), +- ggml_view_1d(ctx, store_c, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32)); + if (!tie) { + tie = pair; + } else { + tie = ggml_add(ctx, +- ggml_view_1d(ctx, tie, 1, 0), +- ggml_view_1d(ctx, pair, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32)); + } + } + return tie; +@@ -2400,8 +2400,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + tie = store_s; + } else { + tie = ggml_add(ctx, +- ggml_view_1d(ctx, tie, 1, 0), +- ggml_view_1d(ctx, store_s, 1, 0)); ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32)); + } + } + return tie; diff --git a/patches/0034-m2-pinned-host.patch b/patches/0034-m2-pinned-host.patch new file mode 100644 index 0000000000000000000000000000000000000000..ce25147d35db533199da99a696e9c557eef5817a --- /dev/null +++ b/patches/0034-m2-pinned-host.patch @@ -0,0 +1,80 @@ +opencoti F5 M7-A (0034) — pinned host buffer for M2 CPU-half KV +opencoti note (2026-05-31, bug-250): hunk regenerated via snapshot-diff against the post-0033 bootstrap so it strict git-applies — the original removed two trailing-whitespace lines absent from the current tree. Content unchanged (pinned cpu_buft resolution); verified against the live r2a tree. + + +Switches the M2 head-split CPU-half allocation from +ggml_backend_cpu_buffer_type() (pageable) to ggml_backend_dev_host_buffer_type(dev) +(CUDA pinned host memory) when the layer is offloaded to a CUDA device. Falls +back to pageable CPU buft when no CUDA dev / no dev_host_buffer_type. + +Why: pageable host memory caps PCIe transfers at ~50% of theoretical +bandwidth because every DMA has to first pin the page or stage through a +driver-side bounce buffer. Pinned host memory unlocks full PCIe bandwidth +on every M2 CPU→GPU stream-back AND is the foundation Rolling KV (F5 M7) +builds on — without pinned residency, the M7 streaming pipeline can't hit +its compute/copy-overlap budget at PCIe 3.0 x8. + +Mechanics: + - Layer scope (~line 270): resolve cpu_buft alongside the existing + layer GPU buft. Read dev_host_buffer_type(dev); fall back to plain + cpu_buffer_type if unsupported. + - Per-stream scope (~line 391): consume the layer-scope cpu_buft + instead of calling ggml_backend_cpu_buffer_type() inline. + +f16 and CPU-only paths byte-identical to pre-0034 (dev_host_buffer_type +returns null → fallback fires → identical alloc). q8_0 + M2 + CUDA paths +get pinned residency. Foundation for #296 (Rolling KV / M7). + +Verification: + - Cosmocc build clean (~80s incremental). + - Smoke at 32k + f16 + M2 frac=0.5 + M0/M1/M3 flags engaged on Gemma 4 + A4B Q4_K_M: server boots clean, /v1/completions returns coherent + tokens (no crash, no DSO mismatch). + - Does NOT fix bug-226 (256k+q8_0 semantic collapse — confirmed + deterministic via 1-sample canary 2026-05-29 21:51, scored 0/1 with + same wrong output 'LJXLD' as the original --no-kv-unified failure; + pinning is orthogonal to the corruption mechanism). + - Held for bug-226 root cause before shipping as default — pinning a + broken stack doesn't fix it but doesn't make it worse either. + +Patch order: applies after 0033-cpy-tie-blck-align. Cosmocc-only edit; +CUDA DSO unchanged (host-side buffer-type resolution, no kernel changes). + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -267,11 +267,20 @@ llama_kv_cache::llama_kv_cache( + + ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type(); + ++ // opencoti F5 M7-A (0034) — resolved at layer scope alongside the ++ // per-layer GPU buft so the M2 CPU-half allocation below (line ~390) ++ // can reach pinned host memory when CUDA is offloaded. Falls back to ++ // pageable CPU buft when no CUDA dev / no dev_host_buffer_type. Pinned ++ // host buffers DMA at full PCIe bandwidth; pageable hits ~50%. ++ ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type(); + if (offload) { + auto * dev = model.dev_layer(il); + buft = ggml_backend_dev_buffer_type(dev); + + dev_name = ggml_backend_dev_name(dev); ++ ++ ggml_backend_buffer_type_t pinned = ggml_backend_dev_host_buffer_type(dev); ++ if (pinned) cpu_buft = pinned; + } + + LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name); +@@ -374,7 +383,12 @@ llama_kv_cache::llama_kv_cache( + // when gpu_heads == 0 so headinfer_split_active() can test + // by vector size. + if (gpu_heads > 0) { +- ggml_context * ctx_cpu_s = ctx_for_buft(ggml_backend_cpu_buffer_type(), s); ++ // opencoti F5 M7-A (0034) — was ggml_backend_cpu_buffer_type() ++ // (pageable). Now uses the layer-scope cpu_buft resolved above: ++ // pinned host buffer when CUDA is the layer dev, plain CPU buft ++ // otherwise. Pinned unlocks full PCIe DMA for M2's CPU-half ++ // stream-back AND is the foundation Rolling KV (M7) builds on. ++ ggml_context * ctx_cpu_s = ctx_for_buft(cpu_buft, s); + if (!ctx_cpu_s) { + throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); + } diff --git a/patches/0035-concat-q-block-aware.patch b/patches/0035-concat-q-block-aware.patch new file mode 100644 index 0000000000000000000000000000000000000000..799de07a5f19994dfe6f4915ac7ed33d88ca9027 --- /dev/null +++ b/patches/0035-concat-q-block-aware.patch @@ -0,0 +1,209 @@ +opencoti F5 bug-226 — block-aware ggml-cpu concat for quantized types + +At 32k+q8_0 KV with M2 head split (--headinfer-gpu-heads-frac < 1.0) +without NEO orchestration, the M2 head-axis concat path in get_k/get_v +(llama-kv-cache.cpp:1881-1910) produced semantically corrupt K/V that +the downstream flash-attn read as garbage, scoring 0.00% on RULER vt +with completely wrong output (e.g. predicting 'SHRUU' instead of the +five expected variables). 5-cell bisection at 32k+q8_0 isolated M2 alone +(--headinfer-gpu-heads-frac 0.5) as the failing config; M0, M1, M3, and +vanilla scored 100. + +Root cause: ggml_concat on block-quantized tensors falls through +multiple broken paths. + +The M2 non-NEO get_k/get_v branch returns: + + ggml_concat(ctx, ggml_view_4d(k_s, head_dim, n_head_gpu, n_kv, 1), + ggml_view_4d(kc_s, head_dim, n_head_cpu, n_kv, 1), 1) + +both views are q8_0. The CUDA backend rejects this (concat.cu:165-167 +asserts F32-only — task #253's earlier supports_op tightening was +correct), so the scheduler dispatches to CPU. CPU concat falls into +ggml_compute_forward_concat_any (ops.cpp:1894-1935) for any non- +F16/BF16/I16/I8/F32/I32 type. concat_any iterates i0 over LOGICAL +elements (0..ne0) with stride nb00, calling memcpy(len = type_size). +For q8_0, nb00 = type_size = 36 bytes (one block of 32 elements), but +the code treats it as a per-element stride — reading blocks at offsets +0, 36, 72, ..., 127·36 = 4572 bytes from each row start, well past +the actual row of (ne0/32)·36 = 144 bytes. Result: q8_0 blocks +scrambled into uninterpretable data → FA reads garbage K → output is +task-format-correct but value-wrong. + +At f16 KV (block size 1) this latent bug never triggered because +concat dispatches to ggml_compute_forward_concat_f16 which correctly +handles fp16 per-element. The full F5 stack at f16 + 32k scored 100 +in the morning RULER comparison, masking the q8_0 case. + +Fix: a new ggml_compute_forward_concat_q that copies entire dim-0 rows +at once for block-quantized types. Dispatched from +ggml_compute_forward_concat's default branch when ggml_blck_size > 1. + +For dim != 0 (the M2 head-axis case): each (i1, i2, i3) maps to either +src0 or src1 (mutually exclusive); one memcpy of (ne0/blck)·type_size +bytes per row. For dim == 0: dst's dim-0 row = src0_row || src1_row, +asserted block-aligned at the split; two memcpys per row. + +Non-block-quantized (F32/F16/BF16/I8/I16/I32) paths unchanged. The +ggml-cuda CUDA path stays F32-only — that's correct since this fix +lands at the CPU fallback the scheduler already routes to. + +Lives in ggml-cpu only — no CUDA DSO rebuild needed. Compatible with +incremental cosmocc rebuild via `OPENCOTI_NO_CCACHE=1 bun run +build:llamafile:make`. + +Verification: +- T2 config smoke (M2-only, no NEO, q8_0, 32k, vt n=1) — bug repro + pre-fix scored 0.00% with pred='SHRUU'; post-fix expected 100.00%. +- Full F5 stack (M0+M1+M2+M3+NEO+Phase4) at q8_0 + 32k — unchanged + from morning baseline (100%). +- All M0/M1/M2/M3/M5 unified-mode regression-shield benches — unchanged + (cosine ≥ 0.999); concat_q only fires on block-quantized types, and + f16/bf16/f32 KV paths route to their own concat_fN as before. + +apply order: after 0034-m2-pinned-host (this is a ggml-cpu fix; the +two are independent so order is bookkeeping only). + +Co-Authored-By: Claude Opus 4.7 (1M context) + +diff --git a/llama.cpp/ggml/src/ggml-cpu/ops.cpp b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp ++++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +@@ -2078,6 +2078,119 @@ static void ggml_compute_forward_concat_f32( + } + } + ++// opencoti F5 bug-226 fix (0035) — block-quantized concat. ++// ++// concat_any (above) does per-logical-element memcpy(len = type_size), ++// stepping i0 from 0..ne0 with stride nb00. For block-quantized types ++// (q8_0, q4_0, …) the parent tensor stores ne0 LOGICAL elements packed ++// in ne0/blck blocks of type_size bytes each, with nb00 = type_size ++// (one block, not one element). concat_any's loop therefore reads ++// blocks at offsets 0, type_size, 2·type_size, … up to ne0·type_size — ++// well past the actual row of (ne0/blck)·type_size bytes. The result ++// is scrambled block data, which the downstream FA reads as garbage K ++// (root cause of bug-226 at M2 head-axis q8_0 K/V concat). ++// ++// Fix: for block-quantized types, copy entire dim-0 rows at once. ++// Each dim-0 row of src0/src1/dst is contiguous (nb00 = type_size, ++// ne0/blck blocks per row, no gaps), so a single memcpy per ++// (i1, i2, i3) selection on each source captures the right bytes. ++// ++// - dim != 0 (M2 head-axis case): per (i1, i2, i3), the whole dim-0 ++// row comes from src0 or src1 (mutually exclusive). One memcpy. ++// - dim == 0 (split on dim 0): dst's dim-0 row = src0's dim-0 row || ++// src1's dim-0 row, concatenated head-to-tail. The split MUST land ++// on a block boundary (ne00 % blck == 0); when it does, two memcpys. ++// ++// The M2 split in llama-kv-cache.cpp:get_k/get_v concats on dim 1 ++// (head axis), so the dim != 0 branch is the hot path. dim 0 is ++// covered for completeness — any future quantized-feature-axis split ++// pattern lands correctly without re-discovering this bug. ++static void ggml_compute_forward_concat_q( ++ const ggml_compute_params * params, ++ ggml_tensor * dst) { ++ ++ const ggml_tensor * src0 = dst->src[0]; ++ const ggml_tensor * src1 = dst->src[1]; ++ ++ const size_t type_size = ggml_type_size(src0->type); ++ const int64_t blck = ggml_blck_size(src0->type); ++ ++ GGML_ASSERT(blck > 1 && "concat_q is for block-quantized types"); ++ GGML_ASSERT(ggml_type_size(src1->type) == type_size); ++ GGML_ASSERT(ggml_blck_size(src1->type) == blck); ++ GGML_ASSERT(ggml_type_size(dst->type) == type_size); ++ GGML_ASSERT(ggml_blck_size(dst->type) == blck); ++ ++ const int ith = params->ith; ++ const int nth = params->nth; ++ ++ GGML_TENSOR_BINARY_OP_LOCALS ++ ++ const int32_t dim = ggml_get_op_params_i32(dst, 0); ++ ++ GGML_ASSERT(dim >= 0 && dim < 4); ++ ++ // Both srcs must have block-aligned dim-0 sizes — guaranteed by ggml ++ // for any well-formed quantized tensor. We assert anyway because if ++ // a caller violates this, the row math below silently corrupts data. ++ GGML_ASSERT((ne00 % blck) == 0); ++ GGML_ASSERT((ne10 % blck) == 0); ++ ++ // Bytes per dim-0 row of each source / dst. ++ const size_t row_bytes_src0 = (ne00 / blck) * type_size; ++ const size_t row_bytes_src1 = (ne10 / blck) * type_size; ++ const size_t row_bytes_dst = (ne0 / blck) * type_size; ++ ++ if (dim == 0) { ++ // dst row = src0 row (head) || src1 row (tail). Both fill at the ++ // block boundary. For non-concat dims i1/i2/i3 must match between ++ // src0 and src1 (ggml concat invariant), so no per-dim mux logic. ++ GGML_ASSERT(row_bytes_src0 + row_bytes_src1 == row_bytes_dst); ++ for (int i3 = 0; i3 < ne3; i3++) { ++ for (int i2 = ith; i2 < ne2; i2 += nth) { ++ for (int i1 = 0; i1 < ne1; i1++) { ++ const char * x0 = (const char *)src0->data ++ + i1*nb01 + i2*nb02 + i3*nb03; ++ const char * x1 = (const char *)src1->data ++ + i1*nb11 + i2*nb12 + i3*nb13; ++ char * y = (char *)dst->data ++ + i1*nb1 + i2*nb2 + i3*nb3; ++ memcpy(y, x0, row_bytes_src0); ++ memcpy(y + row_bytes_src0, x1, row_bytes_src1); ++ } ++ } ++ } ++ return; ++ } ++ ++ // dim != 0: each (i1, i2, i3) maps to either src0 or src1, never both. ++ // row_bytes is shared (src0/src1/dst all have the same dim-0 row). ++ GGML_ASSERT(row_bytes_src0 == row_bytes_src1); ++ GGML_ASSERT(row_bytes_src0 == row_bytes_dst); ++ ++ int64_t o[4] = {0, 0, 0, 0}; ++ o[dim] = src0->ne[dim]; ++ ++ // TODO: smarter multi-threading ++ for (int i3 = 0; i3 < ne3; i3++) { ++ for (int i2 = ith; i2 < ne2; i2 += nth) { ++ for (int i1 = 0; i1 < ne1; i1++) { ++ const char * x; ++ if (i1 < ne01 && i2 < ne02 && i3 < ne03) { ++ x = (const char *)src0->data ++ + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; ++ } else { ++ x = (const char *)src1->data ++ + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; ++ } ++ char * y = (char *)dst->data ++ + i1*nb1 + i2*nb2 + i3*nb3; ++ memcpy(y, x, row_bytes_dst); ++ } ++ } ++ } ++} ++ + void ggml_compute_forward_concat( + const ggml_compute_params * params, + ggml_tensor * dst) { +@@ -2102,7 +2215,16 @@ void ggml_compute_forward_concat( + } break; + default: + { +- ggml_compute_forward_concat_any(params, dst); ++ // opencoti F5 bug-226 fix (0035): block-quantized types ++ // (q8_0, q4_0, q5_0, …) route to concat_q which iterates ++ // over BLOCKS, not logical elements. Without this they ++ // hit concat_any whose per-element memcpy with type_size ++ // = block-size scrambles q8_0 K/V at M2 head-axis concat. ++ if (ggml_blck_size(src0->type) > 1) { ++ ggml_compute_forward_concat_q(params, dst); ++ } else { ++ ggml_compute_forward_concat_any(params, dst); ++ } + } + } + } diff --git a/patches/0036-pcie-probe-consume.patch b/patches/0036-pcie-probe-consume.patch new file mode 100644 index 0000000000000000000000000000000000000000..0405c120e6a189ddc988698b83f3299a79e25001 --- /dev/null +++ b/patches/0036-pcie-probe-consume.patch @@ -0,0 +1,364 @@ +opencoti F5-opt W1 (#293) — PCIe/ReBAR profile boot-time reader + autodetect flags + +WHAT + Consumes the shipped rebar/PCIe probe (perf/llamafile/rebar-probe.{cu,sh}) at + server boot: reads the newest probe JSON (or $OPENCOTI_PCIE_PROFILE) for the + measured pinned H2D bandwidth + PCIe link width/speed, falling back to a + one-shot nvidia-smi query, then a conservative PCIe3 x8 default. The result + (effective GB/s + link geometry) is cached in a process-global pcie_profile, + logged at startup, and consumed later by M7 Rolling KV (#296) for streaming + tile sizing (tile_bytes_max = compute_ms * eff_bw_GBps * 0.8). + + Adds: + - common/pcie-profile.{h,cpp} (NEW — detection + global) + - common_params.pcie_autodetect/pcie_bw_gbps (common.h) + - --pcie-autodetect / --pcie-bw-gbps (common/arg.cpp) + - common/pcie-profile.cpp TU (BUILD.mk COMMON_SRCS_CPP) + - pcie_profile_init() + boot log (tools/server/server.cpp — hook) + +PROVENANCE (per opencoti upstream-sync directive) + opencoti-ORIGINAL — no code copied from upstream llama.cpp. The PCIe-link + sysfs read + ReBAR-active detection technique derives from lightseek's + tools/windows/rebar_test.cu (Windows NVML/VirtualAlloc original), Linux-ported + by opencoti in perf/llamafile/rebar-probe.cu (#293). The server-side reader + here parses that probe's JSON. + Upstream base: ggml-org/llama.cpp via Mozilla-Ocho/llamafile submodule + 5e9c63546 (llamafile 0.10.1 pin). Apply order: after 0035 (independent — + common/ + server additive). Surgical hook: tools/server/server.cpp boot call + — registered in docs/protocols/UPSTREAM_SYNC.md. + + Co-Authored-By: Claude Opus 4.8 (1M context) + +diff -ruN state-pre-0036/llama.cpp/BUILD.mk state-post-0036/llama.cpp/BUILD.mk +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -242,6 +242,7 @@ COMMON_SRCS_CPP := \ + llama.cpp/common/ngram-cache.cpp \ + llama.cpp/common/ngram-map.cpp \ + llama.cpp/common/ngram-mod.cpp \ ++ llama.cpp/common/pcie-profile.cpp \ + llama.cpp/common/peg-parser.cpp \ + llama.cpp/common/preset.cpp \ + llama.cpp/common/reasoning-budget.cpp \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1440,6 +1440,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); ++ // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md ++ add_opt(common_arg( ++ {"--pcie-autodetect"}, "MODE", ++ "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.", ++ [](common_params & params, const std::string & value) { ++ if (value == "on" || value == "true" || value == "1") params.pcie_autodetect = true; ++ else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false; ++ else throw std::invalid_argument("--pcie-autodetect must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT")); ++ add_opt(common_arg( ++ {"--pcie-bw-gbps"}, "F", ++ string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps), ++ [](common_params & params, const std::string & value) { ++ params.pcie_bw_gbps = std::stof(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS")); + add_opt(common_arg( + {"--chunks"}, "N", + string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks), +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -445,6 +445,12 @@ struct common_params { + // active for the layer), 2 = auto (reserved for future load-based + // engagement; same as `on` today). + uint32_t neo_pipeline_mode = 0; ++ // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md ++ // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 ++ // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / ++ // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default). ++ bool pcie_autodetect = true; ++ float pcie_bw_gbps = 0.0f; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/common/pcie-profile.cpp b/llama.cpp/common/pcie-profile.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/common/pcie-profile.cpp +@@ -0,0 +1,201 @@ ++// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile. ++// See pcie-profile.h for the contract. CPU-side only; never throws. ++ ++#include "pcie-profile.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++// PCIe generation -> per-lane raw transfer rate (GT/s). ++float gen_to_gtps(int gen) { ++ switch (gen) { ++ case 1: return 2.5f; ++ case 2: return 5.0f; ++ case 3: return 8.0f; ++ case 4: return 16.0f; ++ case 5: return 32.0f; ++ case 6: return 64.0f; ++ default: return 0.0f; ++ } ++} ++ ++// Effective usable H2D bandwidth from link geometry. The 0.8 factor folds in ++// 128b/130b (gen3+) encoding overhead + DMA/protocol inefficiency; matches the ++// rolling_kv.md auto-adapt formula. width lanes * GT/s / 8 bits-per-byte * 0.8. ++float geometry_bw_gbps(int width, float gtps) { ++ if (width <= 0 || gtps <= 0.0f) return 0.0f; ++ return (float) width * gtps * 0.8f / 8.0f; ++} ++ ++// "8.0 GT/s PCIe" / "16.0 GT/s PCIe" -> leading float. Empty/garbage -> 0. ++float parse_leading_float(const std::string & s) { ++ return s.empty() ? 0.0f : (float) atof(s.c_str()); ++} ++ ++// Minimal value-after-key extraction for our OWN well-formed probe JSON. ++// Returns the substring between key's ':' and the next ',' or '}' / newline. ++bool json_raw_after(const std::string & body, const std::string & key, std::string & out) { ++ const size_t k = body.find(key); ++ if (k == std::string::npos) return false; ++ size_t c = body.find(':', k + key.size()); ++ if (c == std::string::npos) return false; ++ ++c; ++ while (c < body.size() && (body[c] == ' ' || body[c] == '\t')) ++c; ++ size_t e = c; ++ while (e < body.size() && body[e] != ',' && body[e] != '\n' && body[e] != '}') ++e; ++ out = body.substr(c, e - c); ++ return true; ++} ++ ++float json_float_after(const std::string & body, const std::string & key) { ++ std::string raw; ++ if (!json_raw_after(body, key, raw)) return 0.0f; ++ return (float) atof(raw.c_str()); ++} ++ ++// Quoted-string value: trims surrounding quotes/spaces. ++std::string json_string_after(const std::string & body, const std::string & key) { ++ std::string raw; ++ if (!json_raw_after(body, key, raw)) return {}; ++ size_t a = raw.find('"'); ++ if (a == std::string::npos) return {}; ++ size_t b = raw.find('"', a + 1); ++ if (b == std::string::npos) return {}; ++ return raw.substr(a + 1, b - a - 1); ++} ++ ++std::string read_file(const std::string & path) { ++ std::ifstream f(path, std::ios::binary); ++ if (!f.is_open()) return {}; ++ std::ostringstream ss; ++ ss << f.rdbuf(); ++ return ss.str(); ++} ++ ++// Newest ./.opencoti/rebar-probe-*.json. Filenames embed a YYYYMMDD-HHMMSS ++// stamp, so lexicographic max == newest. Empty if none. ++std::string newest_probe_json() { ++ const char * env = getenv("OPENCOTI_PCIE_PROFILE"); ++ if (env && env[0]) return std::string(env); ++ ++ const char * dirs[] = {"./.opencoti", ".opencoti"}; ++ std::string best; ++ for (const char * d : dirs) { ++ DIR * dp = opendir(d); ++ if (!dp) continue; ++ struct dirent * ent; ++ while ((ent = readdir(dp)) != nullptr) { ++ const std::string name = ent->d_name; ++ if (name.rfind("rebar-probe-", 0) != 0) continue; ++ if (name.size() < 5 || name.compare(name.size() - 5, 5, ".json") != 0) continue; ++ const std::string full = std::string(d) + "/" + name; ++ if (best.empty() || name > best.substr(best.find_last_of('/') + 1)) best = full; ++ } ++ closedir(dp); ++ if (!best.empty()) break; ++ } ++ return best; ++} ++ ++// Parse a probe JSON into a profile. Prefers the MEASURED pinned H2D bandwidth ++// over the geometry formula; still records link width/speed for logging. ++bool profile_from_probe_json(const std::string & path, pcie_profile & out) { ++ const std::string body = read_file(path); ++ if (body.empty()) return false; ++ ++ // measured pinned H2D: locate the "pinned" object, then its "h2d". ++ float pinned_h2d = 0.0f; ++ const size_t p = body.find("\"pinned\""); ++ if (p != std::string::npos) { ++ pinned_h2d = json_float_after(body.substr(p), "\"h2d\""); ++ } ++ ++ const int width = atoi(json_string_after(body, "\"pci_link_width_current\"").c_str()); ++ const float gtps = parse_leading_float(json_string_after(body, "\"pci_link_speed_current\"")); ++ ++ if (pinned_h2d <= 0.0f && width <= 0) return false; ++ ++ out.link_width = width; ++ out.link_gtps = gtps; ++ out.effective_bw_gbps = pinned_h2d > 0.0f ? pinned_h2d : geometry_bw_gbps(width, gtps); ++ out.source = "probe-json"; ++ out.detail = "from " + path + (pinned_h2d > 0.0f ? " (measured pinned H2D)" : " (link geometry)"); ++ return out.effective_bw_gbps > 0.0f; ++} ++ ++// One-shot nvidia-smi query for link gen + width. CPU-side, no CUDA link dep. ++bool profile_from_nvidia_smi(pcie_profile & out) { ++ FILE * pp = popen("nvidia-smi --query-gpu=pcie.link.gen.current,pcie.link.width.current " ++ "--format=csv,noheader,nounits -i 0 2>/dev/null", "r"); ++ if (!pp) return false; ++ char buf[256] = {}; ++ const bool got = fgets(buf, sizeof(buf), pp) != nullptr; ++ pclose(pp); ++ if (!got) return false; ++ ++ int gen = 0, width = 0; ++ // format: ", " ++ if (sscanf(buf, " %d , %d", &gen, &width) < 2) return false; ++ const float gtps = gen_to_gtps(gen); ++ const float bw = geometry_bw_gbps(width, gtps); ++ if (bw <= 0.0f) return false; ++ ++ out.link_width = width; ++ out.link_gtps = gtps; ++ out.effective_bw_gbps = bw; ++ out.source = "nvidia-smi"; ++ out.detail = "gen" + std::to_string(gen) + " x" + std::to_string(width) + " (link geometry)"; ++ return true; ++} ++ ++pcie_profile g_pcie_profile; ++bool g_pcie_inited = false; ++ ++} // namespace ++ ++pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps) { ++ pcie_profile prof; // conservative default below ++ ++ if (manual_gbps > 0.0f) { ++ prof.effective_bw_gbps = manual_gbps; ++ prof.source = "manual"; ++ prof.detail = "--pcie-bw-gbps override"; ++ return prof; ++ } ++ ++ if (autodetect) { ++ const std::string js = newest_probe_json(); ++ if (!js.empty() && profile_from_probe_json(js, prof)) return prof; ++ if (profile_from_nvidia_smi(prof)) return prof; ++ } ++ ++ // Conservative default: PCIe 3.0 x8 (common on Ryzen 5600G/5700G hosts where ++ // the iGPU absorbs 8 lanes — see rolling_kv.md). ~6.4 GB/s. ++ prof.link_width = 8; ++ prof.link_gtps = 8.0f; ++ prof.effective_bw_gbps = geometry_bw_gbps(8, 8.0f); ++ prof.source = "default"; ++ prof.detail = autodetect ? "no probe JSON / nvidia-smi; assuming PCIe3 x8" ++ : "autodetect disabled; assuming PCIe3 x8"; ++ return prof; ++} ++ ++const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps) { ++ g_pcie_profile = pcie_profile_detect(autodetect, manual_gbps); ++ g_pcie_inited = true; ++ return g_pcie_profile; ++} ++ ++const pcie_profile & pcie_profile_get() { ++ return g_pcie_profile; ++} +diff --git a/llama.cpp/common/pcie-profile.h b/llama.cpp/common/pcie-profile.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/common/pcie-profile.h +@@ -0,0 +1,38 @@ ++#pragma once ++ ++// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile. ++// ++// Boot-time detection of the GPU's effective host-to-device transfer bandwidth ++// and PCIe link geometry. This is a foundation diagnostic and, more ++// importantly, the input M7 Rolling KV (#296) uses to size streaming KV tiles: ++// tile_bytes_max = compute_ms_per_layer * effective_bw_gbps * 0.8 ++// (see docs/features/rolling_kv.md, "Auto-adapt loop", step 1). ++// ++// Detection is CPU-side only (no CUDA dependency in common/). Sources, in ++// preference order: ++// 1. manual override (--pcie-bw-gbps F, F > 0) ++// 2. explicit probe JSON via $OPENCOTI_PCIE_PROFILE ++// 3. newest ./.opencoti/rebar-probe-*.json (measured pinned H2D + link) ++// 4. one-shot `nvidia-smi` query (pcie.link.gen.current + width.current) ++// 5. conservative default (PCIe 3.0 x8 ~= 6.4 GB/s) ++// Never throws; never fails server boot. ++ ++#include ++ ++struct pcie_profile { ++ float effective_bw_gbps = 0.0f; // usable H2D bandwidth estimate (GB/s) ++ int link_width = 0; // current PCIe lane count (e.g. 8, 16) ++ float link_gtps = 0.0f; // current per-lane transfer rate (GT/s) ++ std::string source = "default"; // manual|probe-json|nvidia-smi|default ++ std::string detail; // human-readable note for logging ++}; ++ ++// Run detection once. manual_gbps > 0 forces source="manual"; autodetect=false ++// skips probe/nvidia-smi discovery and returns the conservative default. ++pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps); ++ ++// Process-global: pcie_profile_init() runs detection, caches, and returns the ++// result; pcie_profile_get() returns the cached value (defaults until init ++// runs). Consumed by M7 Rolling KV (#296) at server boot. ++const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps); ++const pcie_profile & pcie_profile_get(); +diff --git a/llama.cpp/tools/server/server.cpp b/llama.cpp/tools/server/server.cpp +--- a/llama.cpp/tools/server/server.cpp ++++ b/llama.cpp/tools/server/server.cpp +@@ -7,6 +7,7 @@ + #include "arg.h" + #include "build-info.h" + #include "common.h" ++#include "pcie-profile.h" // opencoti F5-opt W1 (#293) + #include "fit.h" + #include "llama.h" + #include "log.h" +@@ -135,6 +136,17 @@ int server_main(int argc, char ** argv, + // struct that contains llama context and inference + server_context ctx_server; + ++ // opencoti F5-opt W1 (#293): detect host<->device bandwidth + PCIe link ++ // once at boot; logged here and cached (pcie_profile_get) for M7 Rolling KV ++ // (#296) streaming-tile sizing. Never fails boot — falls back to a ++ // conservative PCIe3 x8 default on any detection miss. ++ { ++ const pcie_profile & pp = pcie_profile_init(params.pcie_autodetect, params.pcie_bw_gbps); ++ LOG_INF("%s: pcie profile: %.1f GB/s effective (x%d @ %.1f GT/s, source=%s) — %s\n", ++ __func__, (double) pp.effective_bw_gbps, pp.link_width, (double) pp.link_gtps, ++ pp.source.c_str(), pp.detail.c_str()); ++ } ++ + server_http_context ctx_http; + if (!ctx_http.init(params)) { + SRV_ERR("%s", "failed to initialize HTTP server\n"); diff --git a/patches/0040-iqk-flash-attn.patch b/patches/0040-iqk-flash-attn.patch new file mode 100644 index 0000000000000000000000000000000000000000..f99a62fad4137a51864e9d5ad9ff79d837d79655 --- /dev/null +++ b/patches/0040-iqk-flash-attn.patch @@ -0,0 +1,26688 @@ +opencoti F5-opt W2 (#290) — wholesale ik_llama.cpp CPU Flash-Attention engine import + +Vendors ik_llama.cpp's CPU Flash-Attention engine behind a build flag +(GGML_IQK_FLASH_ATTENTION) plus a single surgical dispatch hook, giving the +M2 CPU-half (HeadInfer's CPU-resident attention heads) a faster +token-generation FA kernel. Off by default; --iqk-flash-attn on engages it. +Flag-OFF is byte-identical to the pre-0040 binary: the hook early-returns when +the process-global toggle is false, and the engine TUs are only reachable +through that call site. + + opencoti-hook: f5-opt-cpufa — see docs/features/advanced_kv.md + +UPSTREAM PROVENANCE (for future re-sync) + Source repo : https://github.com/ikawrakow/ik_llama.cpp + Commit SHA : 8960c5ba5ee9db30ba838304373aa4dbec9f7cbd (HEAD, FA files 2026-05-29) + License : MIT + Author : Iwan Kawrakow (Copyright (c) 2024-2025 Iwan Kawrakow) + Related PRs : ik_llama #332 (CPU FA TG threading); #208 (Q8_KV cache type — + rides in with the engine, enabled separately for #291). + Files vendored verbatim under llama.cpp/ggml/src/iqk/ (+ fa/), each carrying a + top banner naming its upstream path; do not edit in place, re-sync from source: + iqk/iqk_flash_attn.cpp ← ik_llama ggml/src/iqk/iqk_flash_attn.cpp + iqk/iqk_fa_dispatch.cpp ← ik_llama ggml/src/iqk/iqk_fa_dispatch.cpp + iqk/fa/iqk_fa_templates.h ← ik_llama ggml/src/iqk/fa/iqk_fa_templates.h + iqk/fa/iqk_fa_{64_64,96_96,128_128,192_128,192_192,256_256,320_256,512_512,576_512}.cpp + ← ik_llama ggml/src/iqk/fa/* (9 head-size instantiations) + iqk/iqk_gemm_{floats,kquants,legacy_quants}.{cpp,h}, iqk_gemm_ktquants.h + ← ik_llama ggml/src/iqk/* (FA dot-product backends) + iqk/iqk_quantize.{cpp,h}, iqk_common.h, iqk_common_extra.h, iqk_config.h, + iqk/iqk_utils.h, iqk_mul_mat.h, iqk_flash_impl.h, iqk_ggml_type_ext.h, + iqk/ggml-common.h ← ik_llama ggml/src/iqk/* + ggml/src support headers + +OPENCOTI ADAPTATIONS (delta from upstream — the re-sync touch points) + - llama.cpp/BUILD.mk: adds the engine TUs to the GGML C++ SRCS list and an + IQK_FA_OBJS per-object rule granting them -Xx86_64-mavx2/-mf16c/-mfma + -DGGML_IQK_FLASH_ATTENTION -Wno-enum-constexpr-conversion (bug-243: clang + rejects the >63 ggml_type casts in the FA switch labels otherwise). ops.cpp.o + and ggml-cpu.c.o also receive -DGGML_IQK_FLASH_ATTENTION for the two hooks. + - llama.cpp/ggml/include/ggml-iqk-flash-attn.h (NEW): the public toggle + (set/get) + 3 engine entry decls, pulling ONLY ggml.h. Core TUs include this, + never iqk_mul_mat.h/iqk_config.h, whose NEON shim collides with + ggml-cpu-impl.h on the aarch64 cosmocc slice (bug-244). + - llama.cpp/ggml/src/ggml-cpu/ops.cpp: dispatch hook in + ggml_compute_forward_flash_attn_ext_f16, gated GGML_IQK_FLASH_ATTENTION && + __x86_64__ (bug-245: engine symbols exist only on the AVX2 x86_64 slice). + Calls iqk_flash_attn_noalibi(); on false, falls through to the mainline + _f16 path unchanged. + - llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c: work-buffer sizing hook + (MAX(cur, iqk_fa_work_buffer_size(...))) in the FLASH_ATTN_EXT work-size case. + - llama.cpp/ggml/src/ggml-iqk-flash-attn.cpp (NEW): the atomic toggle TU. + Intentionally EXCLUDED from IQK_FA_OBJS — it holds no engine/SIMD code and + must not carry the enable guard. + - llama.cpp/common/{arg.cpp,common.h}: --iqk-flash-attn on|off CLI + (LLAMA_EXAMPLE_SERVER-gated, LLAMA_ARG_IQK_FLASH_ATTN) + params.iqk_flash_attn. + +VERIFICATION (2026-05-30, Gemma-4 A4B v6-coder Q4_K_M + Qwen2.5 0.5B, solidPC RTX 3090) + - Flag-OFF off-path identity: neo-pipeline.bench C4 (--neo-pipeline off + byte-identical) PASS; advanced-kv-stack 8/9 (sole fail C5 = pre-existing + bug-207 NEO FP-noise, tracked by #280 — unrelated to this patch, ran iqk OFF). + - Flag-ON quality (RULER vt + niah_single_1 @ 4k/32k, M2 + full F5 stack + + --iqk-flash-attn on): 100% all 8 cells; ON vs OFF wall-time flat. The engine + is correctness-clean; on these prefill-bound RULER cells the CPU-split + residency (not CPU-FA kernel speed) is the perf lever — addressed by M7 + Rolling KV / GPU_RESIDENT default (docs/features/rolling_kv.md Decision 4). + +Captured via snapshot-diff (bug-121): state-pre-0040 = post-0036 vendored tree +(BUILD.mk/common.h/arg.cpp from .opencoti/state-post-0036; ops.cpp from +state-post-0035; ggml-cpu.c from pristine HEAD — no prior patch touches it); +state-post-0040 = this W2 tree. Diffs follow. + +diff -ruN state-pre-0040/llama.cpp/BUILD.mk state-post-0040/llama.cpp/BUILD.mk +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -41,7 +41,23 @@ GGML_SRCS_CPP := \ + llama.cpp/ggml/src/ggml-cpu/unary-ops.cpp \ + llama.cpp/ggml/src/ggml-cpu/vec.cpp \ + llama.cpp/ggml/src/ggml-cpu/amx/amx.cpp \ +- llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp ++ llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp \ ++ llama.cpp/ggml/src/ggml-iqk-flash-attn.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_fa_dispatch.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_gemm_floats.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_gemm_kquants.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.cpp \ ++ llama.cpp/ggml/src/iqk/iqk_quantize.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_64_64.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_96_96.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_128_128.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_192_128.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_192_192.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_256_256.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_320_256.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_512_512.cpp \ ++ llama.cpp/ggml/src/iqk/fa/iqk_fa_576_512.cpp + + GGML_OBJS := \ + $(GGML_SRCS_C:%.c=o/$(MODE)/%.c.o) \ +@@ -518,6 +534,55 @@ o/$(MODE)/llama.cpp/ggml/src/ggml-quants.c.o \ + o/$(MODE)/llama.cpp/ggml/src/ggml-cpu/quants.c.o: \ + private CCFLAGS += -O3 -mgcc + ++# opencoti F5-opt W2 (#290): ik_llama CPU Flash-Attention engine. ++# The iqk engine TUs gate every SIMD path behind IQK_IMPLEMENT (= __AVX2__ || ++# __ARM_FEATURE_DOTPROD). cosmocc's default x86_64 baseline lacks AVX2, so ++# without these flags the whole engine compiles to empty stubs. Give the engine ++# TUs AVX2/F16C/FMA on the x86_64 slice + the master enable GGML_IQK_FLASH_ATTENTION. ++# (Shipped multi-host ARM/AVX-512 dispatch is a later concern; solidPC is AVX2.) ++# The flag TU ggml-iqk-flash-attn.cpp is intentionally EXCLUDED — it is a plain ++# atomic toggle with no engine/SIMD code and must not carry the enable guard. ++IQK_FA_OBJS := \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_fa_dispatch.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_gemm_floats.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/iqk_quantize.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_64_64.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_96_96.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_128_128.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_128.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_192.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_256_256.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_320_256.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_512_512.cpp.o \ ++ o/$(MODE)/llama.cpp/ggml/src/iqk/fa/iqk_fa_576_512.cpp.o ++ ++# opencoti F5-opt W2 (#290): the iqk engine carries ik_llama.cpp's EXTENDED ++# ggml_type tags (Q8_KV=151, Q8_0_R8=208, ...) supplied via the isolation header ++# iqk_ggml_type_ext.h as `#define ((ggml_type)N)`. llamafile's real enum closes at ++# GGML_TYPE_COUNT=42, so clang derives the range [0,63] and -Wenum-constexpr-conversion ++# (a DEFAULT-ERROR diagnostic) rejects casting 151 etc. to ggml_type in the FA ++# template switch labels + constexpr `type` members. The enum is int-backed at ++# runtime so 151 is representable and codegen is correct; these tags are internal ++# iqk dispatch keys that never reach llamafile core (its type_traits[COUNT] table is ++# never indexed by them). We suppress the pedantic constant-expression check on the ++# iqk TUs ONLY — keeping the hard-isolation invariant (ggml.h + GGML_TYPE_COUNT ++# untouched) rather than poisoning the shared enum. ++$(IQK_FA_OBJS): private CCFLAGS += \ ++ -Xx86_64-mavx2 -Xx86_64-mf16c -Xx86_64-mfma -DGGML_IQK_FLASH_ATTENTION \ ++ -Wno-enum-constexpr-conversion ++ ++# opencoti F5-opt W2 (#290): the two core ggml-cpu TUs that carry the surgical ++# dispatch hooks (ops.cpp = the FA dispatch; ggml-cpu.c = the wdata work-size ++# bump) need GGML_IQK_FLASH_ATTENTION defined to compile the hook in. They stay ++# at the -mgcc baseline (NOT -mavx2) — the hook code is a plain function call into ++# the AVX2-compiled iqk engine, no SIMD of its own. Adding ONLY the -D keeps a ++# no-flag build byte-identical (the hook #if-compiles out entirely). ++o/$(MODE)/llama.cpp/ggml/src/ggml-cpu/ops.cpp.o: private CCFLAGS += -DGGML_IQK_FLASH_ATTENTION ++o/$(MODE)/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c.o: private CCFLAGS += -DGGML_IQK_FLASH_ATTENTION ++ + # ============================================================================== + # Tool executables + # ============================================================================== +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -9,6 +9,10 @@ + #include "sampling.h" + #include "speculative.h" + #include "preset.h" ++// opencoti F5-opt W2 (#290): the --iqk-flash-attn handler flips the ggml-cpu ++// process-global that the FA dispatch hook reads. Header carries only the ++// toggle getter/setter (no iqk_config.h drag — see bug-244). ++#include "ggml-iqk-flash-attn.h" + + // fix problem with std::min and std::max + #if defined(_WIN32) +@@ -1440,6 +1444,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); ++ // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--iqk-flash-attn"}, "MODE", ++ "iqk CPU flash-attention (vendored ik_llama.cpp engine): off (default), on. " ++ "Speeds up the M2 CPU-half attention on the token-generation path; falls back " ++ "to the mainline kernel for unsupported (type, shape) cases. x86_64-only.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.iqk_flash_attn = false; ++ else if (value == "on" || value == "true" || value == "1") params.iqk_flash_attn = true; ++ else throw std::invalid_argument("--iqk-flash-attn must be on|off"); ++ ggml_iqk_flash_attn_set_enabled(params.iqk_flash_attn); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_IQK_FLASH_ATTN")); + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--pcie-autodetect"}, "MODE", +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -450,6 +450,11 @@ struct common_params { + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / + // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default). + bool pcie_autodetect = true; ++ // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md ++ // Engage the vendored ik_llama.cpp CPU Flash-Attention engine on the M2 ++ // CPU-half (token-generation path). Off by default; flag-off is byte-identical. ++ // x86_64-only at runtime (the engine compiles only on the AVX2 x86_64 slice). ++ bool iqk_flash_attn = false; + float pcie_bw_gbps = 0.0f; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) +diff --git a/llama.cpp/ggml/include/ggml-iqk-flash-attn.h b/llama.cpp/ggml/include/ggml-iqk-flash-attn.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/include/ggml-iqk-flash-attn.h +@@ -0,0 +1,57 @@ ++// opencoti F5-opt W2 (#290) — ik_llama CPU Flash-Attention enable switch. ++// ++// A process-global on/off toggle for the vendored ik_llama.cpp CPU ++// Flash-Attention engine (ggml/src/iqk/*). Mirrors the ggml-neo-pipeline ++// global/setter pattern so the ggml-cpu dispatch hook in ++// ggml_compute_forward_flash_attn_ext_f16 can read it WITHOUT a header ++// dependency on llama/common. Default OFF: when off, the hook fast-skips and ++// the FA op path is byte-identical to upstream. ++// ++// The flag is flipped once at server startup from the runtime tactic that ++// resolves --iqk-flash-attn (common/arg.cpp). It is a static process-wide ++// toggle (no per-layer state), so a single setter call at init suffices. ++#pragma once ++ ++#include "ggml.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++// Enable / disable the iqk CPU Flash-Attention engine. Off by default. ++GGML_API void ggml_iqk_flash_attn_set_enabled(bool enabled); ++GGML_API bool ggml_iqk_flash_attn_enabled(void); ++ ++// opencoti F5-opt W2 (#290): public entry points of the vendored ik_llama engine, ++// re-declared here so the ggml-cpu dispatch hooks (ops.cpp + ggml-cpu.c) can call ++// them WITHOUT including ggml/src/iqk/iqk_mul_mat.h — that header drags in ++// iqk_config.h whose ARM ggml_vdotq_s32 NEON shim collides with the definition ++// ggml-cpu-impl.h already provides in those TUs (bug-244). These signatures MUST ++// stay byte-for-byte identical to ggml/src/iqk/iqk_mul_mat.h (extern "C", so the ++// linker matches by name only — a silent mismatch would corrupt args). Re-sync ++// both on any upstream ik_llama bump. ++typedef void (*barrier_t)(void *); ++ ++// Scratch (params->wdata) bytes the engine needs for one FA op at `nthread` threads. ++size_t iqk_fa_work_buffer_size(const struct ggml_tensor * dst, int nthread); ++ ++// Compute flash-attention for the non-alibi case. Returns true if it handled the ++// op (result written to `qkv`); false if the (type, shape) case is unsupported or ++// max_bias>0 (alibi) — caller must then run the mainline FA path. ++bool iqk_flash_attn_noalibi(int type_q, int type_mask, float max_bias, ++ int neq3, int neq2, long nbq3, long nbq2, ++ int nek3, int nek2, long nbk3, long nbk2, ++ int nev3, int nev2, long nbv3, long nbv2, ++ int ne2, int ne1, long nb1, ++ int type_k, int type_v, int Dk, int Dv, ++ int nq, int nk, ++ int stride_q, int stride_k, int stride_v, int stride_m, ++ const void * q, const void * k, const void * v, ++ const void * mask, const void * sinks, ++ float scale, float softcap, float * qkv, ++ void * work_buffer, barrier_t barrier, void * barrier_data, ++ int ith, int nth, int n_swa); ++ ++#ifdef __cplusplus ++} ++#endif +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -15,6 +15,15 @@ + #include "ggml.h" + #include "common.h" + ++#if defined(GGML_IQK_FLASH_ATTENTION) ++// opencoti F5-opt W2 (#290): iqk_fa_work_buffer_size() — sizes the wdata scratch ++// the vendored ik_llama CPU Flash-Attention engine needs (graph-plan side of the ++// ops.cpp dispatch hook). Declared in ggml-iqk-flash-attn.h, NOT iqk/iqk_mul_mat.h, ++// to avoid iqk_config.h's ARM ggml_vdotq_s32 shim colliding with ggml-cpu-impl.h ++// (bug-244). Gated behind GGML_IQK_FLASH_ATTENTION (per-object on ggml-cpu.c.o). ++#include "ggml-iqk-flash-attn.h" ++#endif ++ + #if defined(_MSC_VER) || defined(__MINGW32__) + #include // using malloc.h with MSC/MINGW + #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) +@@ -2969,6 +2978,16 @@ struct ggml_cplan ggml_graph_plan( + size_t decode = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV)); + + cur += MAX(prefill, decode); ++#if defined(GGML_IQK_FLASH_ATTENTION) && defined(__x86_64__) ++ // opencoti F5-opt W2 (#290): reserve enough wdata for the ++ // vendored ik_llama CPU-FA engine so it never overruns when ++ // --iqk-flash-attn is on. Scratch sizing only — does not ++ // affect outputs, so flag-OFF decode stays byte-identical. ++ // Mirrors ik_llama's graph-plan reservation. x86_64-only: ++ // matches the dispatch hook in ops.cpp (the engine symbols ++ // only exist on the AVX2 x86_64 slice — see that hook). ++ cur = MAX(cur, iqk_fa_work_buffer_size(node, n_tasks)); ++#endif + } break; + case GGML_OP_FLASH_ATTN_BACK: + { +diff --git a/llama.cpp/ggml/src/ggml-cpu/ops.cpp b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp ++++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +@@ -8,6 +8,17 @@ + #include "unary-ops.h" + #include "vec.h" + ++#if defined(GGML_IQK_FLASH_ATTENTION) ++// opencoti F5-opt W2 (#290): vendored ik_llama.cpp CPU Flash-Attention engine. ++// Gated entirely behind GGML_IQK_FLASH_ATTENTION (set per-object on ops.cpp.o in ++// BUILD.mk) plus a runtime toggle, so a build without the flag is byte-identical. ++// This header carries BOTH the runtime toggle getter AND the engine entry decls ++// (iqk_flash_attn_noalibi/barrier_t) — we deliberately do NOT include ++// iqk/iqk_mul_mat.h, which would drag in iqk_config.h's ARM ggml_vdotq_s32 shim ++// and collide with ggml-cpu-impl.h (bug-244). ++#include "ggml-iqk-flash-attn.h" ++#endif ++ + #include + #include + #include +@@ -9068,6 +9079,51 @@ static void ggml_compute_forward_flash_attn_ext_f16( + const int ith = params->ith; + const int nth = params->nth; + ++#if defined(GGML_IQK_FLASH_ATTENTION) && defined(__x86_64__) ++ // opencoti-hook: f5-opt-cpufa — see docs/features/advanced_kv.md ++ // x86_64-only: the iqk engine defines its symbols under IQK_IMPLEMENT ++ // (__AVX2__ || __ARM_FEATURE_DOTPROD). Our per-object flags give the iqk TUs ++ // -Xx86_64-mavx2, so the impl exists on the cosmocc x86_64 slice but NOT the ++ // aarch64 slice (cosmocc's ARM baseline has no dotprod). Gating the call on ++ // __x86_64__ keeps it where the symbol lives; ARM falls through to mainline FA. ++ // Vendored ik_llama.cpp CPU Flash-Attention engine. When enabled at runtime ++ // and the (type, shape) case is supported, it computes the whole op into ++ // dst->data and returns true; on false (unsupported case, or alibi via ++ // max_bias>0 — "noalibi") we fall through to the mainline path below, ++ // unchanged. Arg mapping mirrors ik_llama's iqk_flash_attn_noalibi call site ++ // (their ggml.c); llamafile renames params->shared -> params->threadpool. ++ // n_swa = op_params[4], which is 0 in llamafile (mainline never sets it), so ++ // the mask carries any sliding-window structure. The wdata scratch is sized ++ // by the matching iqk_fa_work_buffer_size() bump in ggml-cpu.c's graph plan. ++ if (ggml_iqk_flash_attn_enabled()) { ++ const ggml_tensor * iqk_mask = dst->src[3]; ++ const ggml_tensor * iqk_sinks = dst->src[4]; ++ float iqk_scale, iqk_max_bias, iqk_softcap; ++ memcpy(&iqk_scale, (const float *) dst->op_params + 0, sizeof(float)); ++ memcpy(&iqk_max_bias, (const float *) dst->op_params + 1, sizeof(float)); ++ memcpy(&iqk_softcap, (const float *) dst->op_params + 2, sizeof(float)); ++ if (iqk_softcap != 0.0f) { ++ iqk_scale /= iqk_softcap; ++ } ++ if (iqk_flash_attn_noalibi( ++ q->type, iqk_mask ? iqk_mask->type : GGML_TYPE_F16, iqk_max_bias, ++ (int) neq3, (int) neq2, (long) nbq3, (long) nbq2, ++ (int) nek3, (int) nek2, (long) nbk3, (long) nbk2, ++ (int) nev3, (int) nev2, (long) nbv3, (long) nbv2, ++ (int) ne2, (int) ne1, (long) nb1, ++ k->type, v->type, (int) DK, (int) DV, ++ (int) neq1, (int) nek1, ++ (int) nbq1, (int) nbk1, (int) nbv1, iqk_mask ? (int) iqk_mask->nb[1] : 0, ++ q->data, k->data, v->data, ++ iqk_mask ? iqk_mask->data : NULL, iqk_sinks ? iqk_sinks->data : NULL, ++ iqk_scale, iqk_softcap, (float *) dst->data, ++ params->wdata, (barrier_t) ggml_barrier, (void *) params->threadpool, ++ ith, nth, dst->op_params[4])) { ++ return; ++ } ++ } ++#endif ++ + // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking) + const bool use_ref = params->use_ref; + +diff --git a/llama.cpp/ggml/src/ggml-iqk-flash-attn.cpp b/llama.cpp/ggml/src/ggml-iqk-flash-attn.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-iqk-flash-attn.cpp +@@ -0,0 +1,19 @@ ++// opencoti F5-opt W2 (#290) — ik_llama CPU Flash-Attention enable switch impl. ++// See ggml/include/ggml-iqk-flash-attn.h. Pure ggml; no llama/common include, ++// so ggml-cpu/ops.cpp can read the toggle inside the FA dispatch hook. ++ ++#include "ggml-iqk-flash-attn.h" ++ ++#include ++ ++namespace { ++std::atomic g_iqk_fa_enabled{false}; ++} // namespace ++ ++void ggml_iqk_flash_attn_set_enabled(bool enabled) { ++ g_iqk_fa_enabled.store(enabled, std::memory_order_relaxed); ++} ++ ++bool ggml_iqk_flash_attn_enabled(void) { ++ return g_iqk_fa_enabled.load(std::memory_order_relaxed); ++} +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_128_128.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_128_128.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_128_128.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_128_128) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<128, 128, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<128, 128, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<128, 128, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<128, 128, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<128, 128, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_128.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_128.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_128.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_192_128) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<192, 128, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<192, 128, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<192, 128, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<192, 128, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<192, 128, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_192.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_192.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_192_192.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_192_192) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<192, 192, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<192, 192, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<192, 192, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<192, 192, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<192, 192, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_256_256.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_256_256.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_256_256.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_256_256) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<256, 256, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<256, 256, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<256, 256, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<256, 256, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<256, 256, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_320_256.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_320_256.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_320_256.cpp +@@ -0,0 +1,142 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++namespace { ++ ++template ++inline void iqk_deepseek_helper(KHelper& kh, VHelper& vh, ++ int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ const float * q, const char * mask, float scale, float softcap, float * qkv, ++ const float * sinkf, float * M, float * S) { ++ auto update = [&nq1, &mask, &q, &qkv, &M, &S, stride_q, stride_m, stride_qkv] (int n) { ++ nq1 -= n; ++ if (nq1 == 0) return true; ++ q += n*stride_q; ++ mask += n*stride_m; ++ qkv += n*stride_qkv; ++ if (M && S) { M += n; S += n; } ++ return false; ++ }; ++ if (nq1 >= 16) { ++ int n_step = nq1/16; ++ FlashAttn<320, 256, 16, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 16*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(16*n_step)) return; ++ } ++ if (nq1 >= 8) { ++ int n_step = nq1/8; ++ FlashAttn<320, 256, 8, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 8*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(8*n_step)) return; ++ } ++ if (nq1 >= 4) { ++ int n_step = nq1/4; ++ FlashAttn<320, 256, 4, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 4*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(4*n_step)) return; ++ } ++ if (nq1 == 3) { ++ FlashAttn<320, 256, 3, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 3, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } ++ else if (nq1 == 2) { ++ FlashAttn<320, 256, 2, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 2, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } else { ++ FlashAttn<320, 256, 1, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 1, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } ++} ++ ++template ++inline bool iqk_deepseek_helper(ggml_type type_k, ++ int nq1, int nk1, int stride_q, int stride_k, int stride_v, int stride_m, int stride_qkv, ++ const float * q, const char * k, const char * v, const char * mask, ++ float scale, float softcap, float * qkv, const float * sinkf, float * M, float * S) { ++ if (type_k == GGML_TYPE_Q8_0) { ++ HelperQ80 kh((const char *)k, stride_k); ++ HelperQ80 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q8_0_R8) { ++ HelperQ80R8<320> kh((const char *)k, stride_k); ++ HelperQ80 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q6_0) { ++ HelperQ60 kh((const char *)k, stride_k); ++ HelperQ60 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#if GGML_IQK_FA_ALL_QUANTS ++ if (type_k == GGML_TYPE_Q8_KV) { ++ HelperQ8KV<320> kh((const char *)k, stride_k); ++ HelperQ8KV<256> vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q4_0) { ++ HelperQ40 kh((const char *)k, stride_k); ++ HelperQ40 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q4_1) { ++ HelperQ41 kh((const char *)k, stride_k); ++ HelperQ41 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_IQ4_NL) { ++ HelperIQ4nl kh((const char *)k, stride_k); ++ HelperIQ4nl vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ if (type_k == GGML_TYPE_F16) { ++ HelperF16 kh((const char *)k, stride_k); ++ HelperF16 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ HelperBF16<320, step_k> kh((const char *)k, stride_k); ++ HelperBF16<256, step_k> vh((const char *)v, stride_v); ++ if (nq1 % 8 == 0) { ++ FlashAttnBF16<320, 256, 8, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } else { ++ FlashAttnBF16<320, 256, 1, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } ++ return true; ++ } ++#endif ++ return false; ++} ++ ++} ++ ++IQK_FA_CASE(iqk_fa_320_256) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ if (!(type_k == type_v || (type_k == GGML_TYPE_Q8_0_R8 && type_v == GGML_TYPE_Q8_0))) { ++ return false; ++ } ++ stride_q /= sizeof(float); // q stride as float ++ return iqk_deepseek_helper<32>(type_k, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, (const char *)k, (const char *)v, (const char *)mask, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_512_512.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_512_512.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_512_512.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_512_512) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<512, 512, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<512, 512, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<512, 512, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<512, 512, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<512, 512, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_576_512.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_576_512.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_576_512.cpp +@@ -0,0 +1,142 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++namespace { ++ ++template ++inline void iqk_deepseek_helper(KHelper& kh, VHelper& vh, ++ int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ const float * q, const char * mask, float scale, float softcap, float * qkv, ++ const float * sinkf, float * M, float * S) { ++ auto update = [&nq1, &mask, &q, &qkv, &M, &S, stride_q, stride_m, stride_qkv] (int n) { ++ nq1 -= n; ++ if (nq1 == 0) return true; ++ q += n*stride_q; ++ mask += n*stride_m; ++ qkv += n*stride_qkv; ++ if (M && S) { M += n; S += n; } ++ return false; ++ }; ++ if (nq1 >= 16) { ++ int n_step = nq1/16; ++ FlashAttn<576, 512, 16, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 16*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(16*n_step)) return; ++ } ++ if (nq1 >= 8) { ++ int n_step = nq1/8; ++ FlashAttn<576, 512, 8, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 8*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(8*n_step)) return; ++ } ++ if (nq1 >= 4) { ++ int n_step = nq1/4; ++ FlashAttn<576, 512, 4, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 4*n_step, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ if (update(4*n_step)) return; ++ } ++ if (nq1 == 3) { ++ FlashAttn<576, 512, 3, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 3, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } ++ else if (nq1 == 2) { ++ FlashAttn<576, 512, 2, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 2, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } else { ++ FlashAttn<576, 512, 1, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 1, nk1, stride_q, stride_m, stride_qkv, q, mask, qkv, M, S); ++ } ++} ++ ++template ++inline bool iqk_deepseek_helper(ggml_type type_k, ++ int nq1, int nk1, int stride_q, int stride_k, int stride_v, int stride_m, int stride_qkv, ++ const float * q, const char * k, const char * v, const char * mask, ++ float scale, float softcap, float * qkv, const float * sinkf, float * M, float * S) { ++ if (type_k == GGML_TYPE_Q8_0) { ++ HelperQ80 kh((const char *)k, stride_k); ++ HelperQ80 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q8_0_R8) { ++ HelperQ80R8<576> kh((const char *)k, stride_k); ++ HelperQ80 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q6_0) { ++ HelperQ60 kh((const char *)k, stride_k); ++ HelperQ60 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#if GGML_IQK_FA_ALL_QUANTS ++ if (type_k == GGML_TYPE_Q8_KV) { ++ HelperQ8KV<576> kh((const char *)k, stride_k); ++ HelperQ8KV<512> vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q4_0) { ++ HelperQ40 kh((const char *)k, stride_k); ++ HelperQ40 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_Q4_1) { ++ HelperQ41 kh((const char *)k, stride_k); ++ HelperQ41 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ if (type_k == GGML_TYPE_IQ4_NL) { ++ HelperIQ4nl kh((const char *)k, stride_k); ++ HelperIQ4nl vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ if (type_k == GGML_TYPE_F16) { ++ HelperF16 kh((const char *)k, stride_k); ++ HelperF16 vh((const char *)v, stride_v); ++ iqk_deepseek_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ HelperBF16<576, step_k> kh((const char *)k, stride_k); ++ HelperBF16<512, step_k> vh((const char *)v, stride_v); ++ if (nq1 % 8 == 0) { ++ FlashAttnBF16<576, 512, 8, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } else { ++ FlashAttnBF16<576, 512, 1, step_k> fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } ++ return true; ++ } ++#endif ++ return false; ++} ++ ++} ++ ++IQK_FA_CASE(iqk_fa_576_512) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ if (!(type_k == type_v || (type_k == GGML_TYPE_Q8_0_R8 && type_v == GGML_TYPE_Q8_0))) { ++ return false; ++ } ++ stride_q /= sizeof(float); // q stride as float ++ return iqk_deepseek_helper<32>(type_k, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, (const char *)k, (const char *)v, (const char *)mask, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_64_64.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_64_64.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_64_64.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_64_64) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<64, 64, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<64, 64, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<64, 64, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<64, 64, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<64, 64, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_96_96.cpp b/llama.cpp/ggml/src/iqk/fa/iqk_fa_96_96.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_96_96.cpp +@@ -0,0 +1,45 @@ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include "iqk/fa/iqk_fa_templates.h" ++ ++IQK_FA_CASE(iqk_fa_96_96) { ++ ++ auto type_k = ggml_type(int_type_k); ++ auto type_v = ggml_type(int_type_v); ++ ++ stride_q /= sizeof(float); // q stride as float ++ auto ck = (const char *)k; ++ auto cv = (const char *)v; ++ auto cm = (const char *)mask; ++ ++#ifdef __AVX512BF16__ ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types ++ if (nk%64 == 0) { ++ iqk_flash_helper_T<96, 96, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++ iqk_flash_helper_T<96, 96, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ return true; ++ } ++#endif ++ ++ if (nk%128 == 0) { ++ return iqk_flash_helper_T<96, 96, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ if (nk%64 == 0) { ++ return iqk_flash_helper_T<96, 96, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ } ++ ++ return iqk_flash_helper_T<96, 96, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S); ++ ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/fa/iqk_fa_templates.h b/llama.cpp/ggml/src/iqk/fa/iqk_fa_templates.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/fa/iqk_fa_templates.h +@@ -0,0 +1,2258 @@ ++// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*- ++// vi: set et ft=cpp fenc=utf-8 :vi ++// ++// ++// Copyright (C) 2024 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++ ++#include "iqk/iqk_config.h" ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include ++#include ++#include ++ ++#include "ggml-impl.h" ++#include "ggml-quants.h" ++#include "iqk/iqk_quantize.h" ++#include "iqk/iqk_gemm_floats.h" ++#include "iqk/iqk_gemm_kquants.h" ++#include "iqk/iqk_gemm_legacy_quants.h" ++#include "iqk/iqk_utils.h" ++ ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++ ++// clang-format off ++ ++namespace { ++ ++// Compute effective K boundary by scanning ALL query rows (union-of-masks). ++// The original early-termination scanned only the last row, which is correct ++// for single-slot (all rows have the same mask) but wrong for multi-slot ++// parallel (--parallel N>1) where different slots have different sequence ++// lengths and therefore different mask patterns. ++// Returns the number of K elements to process (multiple of k_step). ++inline int mask_effective_nk1(const char * mask, int n_rows, int stride_m, int nk1, int k_step) { ++ int ik_max = 0; ++ for (int j = 0; j < n_rows; ++j) { ++ auto Mc = (const uint16_t *)(mask + j * stride_m); ++ int ik = nk1 - k_step; ++ for (; ik >= 0 && Mc[ik] != 0; ik -= k_step); ++ ik += k_step; ++ if (ik > ik_max) ik_max = ik; ++ } ++ return ik_max; ++} ++ ++struct BaseHelper { ++ BaseHelper(const char * data, int stride) : data(data), block(data), stride(stride) {} ++ ++ //inline void set_block(int k1) { block = data + k1*k_step*stride; } ++ inline void reset_block() { block = data; } ++ inline void next_block(int step) { block += step*stride; } ++ inline const char * lblock(int l1) const { return block + l1*stride; } ++ ++ const char * data; ++ const char * block; ++ int stride; ++ ++}; ++ ++struct F16 { ++#ifdef __AVX512F__ ++ using Data = __m512; ++ constexpr static int block_size = 16; ++ constexpr static int num_registers = 32; ++ constexpr static int q_step = 8; ++ static inline Data zero() { return _mm512_setzero_ps(); } ++ static inline Data load(const char * ptr, int i) { return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)ptr + i)); } ++ static inline Data set1(float val) { return _mm512_set1_ps(val); } ++ static inline Data mul(Data v1, Data v2) { return _mm512_mul_ps(v1, v2); } ++ static inline Data sub(Data v1, Data v2) { return _mm512_sub_ps(v1, v2); } ++ static inline Data load(const float * ptr) { return _mm512_loadu_ps(ptr); } ++ static inline void store(float * ptr, Data data) { _mm512_storeu_ps(ptr, data); } ++ static inline Data fmadd(Data prev, Data v1, Data v2) { return _mm512_fmadd_ps(v1, v2, prev); } ++ static inline float reduce_max(Data data) { return _mm512_reduce_max_ps(data); } ++ static inline float reduce_add(Data data) { return _mm512_reduce_add_ps(data); } ++ static inline Data max(Data v1, Data v2) { return _mm512_max_ps(v1, v2); } ++ static inline Data add(Data v1, Data v2) { return _mm512_add_ps(v1, v2); } ++ static inline Data set4(const float * ptr) { ++ auto v128 = _mm_loadu_ps(ptr); ++ auto v256 = _mm256_set_m128(v128, v128); ++ return _mm512_insertf32x8(_mm512_castps256_ps512(v256), v256, 1); ++ } ++ static inline void set4(const float * ptr, Data * vs) { ++ auto v = set4(ptr); ++ vs[0] = _mm512_shuffle_ps(v, v, 0x00); ++ vs[1] = _mm512_shuffle_ps(v, v, 0x55); ++ vs[2] = _mm512_shuffle_ps(v, v, 0xaa); ++ vs[3] = _mm512_shuffle_ps(v, v, 0xff); ++ } ++ static inline Data fmadd_lane0(Data prev, Data v1, Data v2) { return _mm512_fmadd_ps(v1, _mm512_shuffle_ps(v2, v2, 0x00), prev); } ++ static inline Data fmadd_lane1(Data prev, Data v1, Data v2) { return _mm512_fmadd_ps(v1, _mm512_shuffle_ps(v2, v2, 0x55), prev); } ++ static inline Data fmadd_lane2(Data prev, Data v1, Data v2) { return _mm512_fmadd_ps(v1, _mm512_shuffle_ps(v2, v2, 0xaa), prev); } ++ static inline Data fmadd_lane3(Data prev, Data v1, Data v2) { return _mm512_fmadd_ps(v1, _mm512_shuffle_ps(v2, v2, 0xff), prev); } ++#elif defined __AVX2__ ++ using Data = __m256; ++ constexpr static int block_size = 8; ++ constexpr static int num_registers = 16; ++ constexpr static int q_step = 8; ++ static inline Data zero() { return _mm256_setzero_ps(); } ++ static inline Data load(const char * ptr, int i) { return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)ptr + i)); } ++ static inline Data set1(float val) { return _mm256_set1_ps(val); } ++ static inline Data mul(Data v1, Data v2) { return _mm256_mul_ps(v1, v2); } ++ static inline Data load(const float * ptr) { return _mm256_loadu_ps(ptr); } ++ static inline Data sub(Data v1, Data v2) { return _mm256_sub_ps(v1, v2); } ++ static inline void store(float * ptr, Data data) { _mm256_storeu_ps(ptr, data); } ++ static inline Data fmadd(Data prev, Data v1, Data v2) { return _mm256_fmadd_ps(v1, v2, prev); } ++ static inline float reduce_max(Data data) { return hmax_float_8(data); } ++ static inline float reduce_add(Data data) { return hsum_float_8(data); } ++ static inline Data max(Data v1, Data v2) { return _mm256_max_ps(v1, v2); } ++ static inline Data add(Data v1, Data v2) { return _mm256_add_ps(v1, v2); } ++ static inline Data set4(const float * ptr) { ++ auto v128 = _mm_loadu_ps(ptr); ++ return _mm256_set_m128(v128, v128); ++ } ++ static inline void set4(const float * ptr, Data * vs) { ++ auto v = set4(ptr); ++ vs[0] = _mm256_shuffle_ps(v, v, 0x00); ++ vs[1] = _mm256_shuffle_ps(v, v, 0x55); ++ vs[2] = _mm256_shuffle_ps(v, v, 0xaa); ++ vs[3] = _mm256_shuffle_ps(v, v, 0xff); ++ } ++ static inline Data fmadd_lane0(Data prev, Data v1, Data v2) { return _mm256_fmadd_ps(v1, _mm256_shuffle_ps(v2, v2, 0x00), prev); } ++ static inline Data fmadd_lane1(Data prev, Data v1, Data v2) { return _mm256_fmadd_ps(v1, _mm256_shuffle_ps(v2, v2, 0x55), prev); } ++ static inline Data fmadd_lane2(Data prev, Data v1, Data v2) { return _mm256_fmadd_ps(v1, _mm256_shuffle_ps(v2, v2, 0xaa), prev); } ++ static inline Data fmadd_lane3(Data prev, Data v1, Data v2) { return _mm256_fmadd_ps(v1, _mm256_shuffle_ps(v2, v2, 0xff), prev); } ++#else ++ using Data = float16x8_t; ++ constexpr static int block_size = 8; ++ //constexpr static int num_registers = 32; ++ //constexpr static int q_step = 8; ++ static inline Data zero() { return vdupq_n_f16(0); } ++ static inline Data load(const char * ptr, int i) { return vld1q_f16((const float16_t *)ptr + block_size*i); } ++ static inline Data load(const float16_t * ptr, int i) { return vld1q_f16(ptr + block_size*i); } ++ static inline Data load(const float16_t * ptr) { return vld1q_f16(ptr); } ++ static inline Data load(const float * ptr) { ++ auto val1 = vld1q_f32(ptr); ++ auto val2 = vld1q_f32(ptr+4); ++ return vcombine_f16(vcvt_f16_f32(val1), vcvt_f16_f32(val2)); ++ } ++ static inline Data set1(float val) { return vdupq_n_f16(val); } ++ static inline Data mul(Data v1, Data v2) { return vmulq_f16(v1, v2); } ++ static inline Data sub(Data v1, Data v2) { return vsubq_f16(v1, v2); } ++ static inline void store(float * ptr, Data data) { ++ vst1q_f32(ptr+0, vcvt_f32_f16(vget_low_f16(data))); ++ vst1q_f32(ptr+4, vcvt_f32_f16(vget_high_f16(data))); ++ } ++ static inline void store(float16_t * ptr, Data data) { vst1q_f16(ptr, data); } ++ static inline void store(float * ptr, float32x4_t data) { vst1q_f32(ptr, data); } ++ static inline Data fmadd(Data prev, Data v1, Data v2) { return vfmaq_f16(prev, v1, v2); } ++ static inline float reduce_max(Data data) { return vmaxvq_f16(data); } ++ static inline float reduce_add(Data data) { ++ auto sum = vadd_f16(vget_low_f16(data), vget_high_f16(data)); ++ return vaddvq_f32(vcvt_f32_f16(sum)); ++ } ++ static inline Data max(Data v1, Data v2) { return vmaxq_f16(v1, v2); } ++ static inline Data add(Data v1, Data v2) { return vaddq_f16(v1, v2); } ++ static inline float16x4_t set4(const float * ptr) { ++ auto val32 = vld1q_f32(ptr); ++ return vcvt_f16_f32(val32); ++ } ++ static inline Data fmadd_lane0(Data prev, Data v1, float16x4_t v2) { return vfmaq_lane_f16(prev, v1, v2, 0); } ++ static inline Data fmadd_lane1(Data prev, Data v1, float16x4_t v2) { return vfmaq_lane_f16(prev, v1, v2, 1); } ++ static inline Data fmadd_lane2(Data prev, Data v1, float16x4_t v2) { return vfmaq_lane_f16(prev, v1, v2, 2); } ++ static inline Data fmadd_lane3(Data prev, Data v1, float16x4_t v2) { return vfmaq_lane_f16(prev, v1, v2, 3); } ++#endif ++ template static inline float reduce_max(const Data * data) { ++ return reduce_T(data); ++ } ++ template static inline float reduce_add(const Data * data) { ++ return reduce_T(data); ++ } ++ template ++ static float reduce_T(const Data * data) { ++ float result; ++ if constexpr (k_step/block_size == 1) { ++ result = Op(data[0]); ++ } ++ else if constexpr (k_step/block_size == 2) { ++ result = Op(Op_combine(data[0], data[1])); ++ } ++ else { ++ auto vmax = Op_combine(data[0], data[1]); ++ for (int l = 2; l < k_step/block_size; ++l) vmax = Op_combine(vmax, data[l]); ++ result = Op(vmax); ++ } ++ return result; ++ } ++}; ++ ++struct HelperF16 final : public BaseHelper { ++ using Base = BaseHelper; ++ HelperF16(const char * data, int stride) : Base(data, stride) {} ++ ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ //auto dr = (const ggml_half *)Base::lblock(l1); ++ auto dr = Base::lblock(l1); ++ v1 = F16::load(dr, i + 0); ++ v2 = F16::load(dr, i + 1); ++ } ++}; ++ ++template struct block_q8_KV { ++ float d; ++ int s; ++ int8_t qs[D]; ++}; ++ ++template ++struct HelperQ8KV final : public BaseHelper { ++ using Base = BaseHelper; ++ using block_q8 = block_q8_KV; ++ constexpr static ggml_type type = GGML_TYPE_Q8_KV; ++ constexpr static int block_size_q = D; ++ HelperQ8KV(const char * data, int stride) : Base(data, stride) {} ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ auto q8 = (const block_q8_KV *)Base::lblock(l1); ++#ifdef __aarch64__ ++ auto vd = F16::set1(q8->d); ++ auto qs = vld1_s8_x2(q8->qs + 8*i); ++ v1 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(qs.val[0]))); ++ v2 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(qs.val[1]))); ++#else ++ auto vd = F16::set1(q8->d); ++#ifdef __AVX512F__ ++ v1 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)q8->qs+i+0)))); ++ v2 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)q8->qs+i+1)))); ++#else ++ v1 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(q8->qs+8*i+0))))); ++ v2 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(q8->qs+8*i+8))))); ++#endif ++#endif ++ } ++}; ++ ++struct HelperQ80 final : public BaseHelper { ++ using Base = BaseHelper; ++ constexpr static ggml_type type = GGML_TYPE_Q8_0; ++//#ifdef HAVE_FANCY_SIMD ++#ifdef __AVX2__ ++ using block_q8 = block_q8_2; ++ constexpr static int block_size_q = QK8_2; ++#else ++ using block_q8 = block_q8_0; ++ constexpr static int block_size_q = QK8_0; ++#endif ++ HelperQ80(const char * data, int stride) : Base(data, stride) {} ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ int j = F16::block_size*i; ++ auto dl = (const block_q8_0 *)Base::lblock(l1) + j/QK8_0; ++#ifdef __aarch64__ ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++ int ii = j%QK8_0; ++ auto qs = vld1_s8_x2(dl->qs + ii); ++ v1 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(qs.val[0]))); ++ v2 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(qs.val[1]))); ++#else ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++#ifdef __AVX512F__ ++ v1 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)dl->qs+0)))); ++ v2 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)dl->qs+1)))); ++#else ++ int ii = j%QK8_0; ++ v1 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(dl->qs+ii+0))))); ++ v2 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(dl->qs+ii+8))))); ++#endif ++#endif ++ } ++ ++ template ++ static inline void convert(int nq, int stride_q, const float * q, block_q8_0 * y) { ++ for (int i = 0; i < nq; ++i) { ++ quantize_row_q8_0_x4(q, y, D); ++ q += stride_q; ++ y += D/QK8_0; ++ } ++ } ++ ++ template ++ static inline void convert(int nq, int stride_q, const float * q, block_q8_1 * y) { ++ for (int i = 0; i < nq; ++i) { ++ quantize_row_q8_1_x4(q, y, D); ++ q += stride_q; ++ y += D/QK8_1; ++ } ++ } ++ ++ template ++ static inline void convert(int nq, int stride_q, const float * q, block_q8_2 * y) { ++ for (int i = 0; i < nq; ++i) { ++ quantize_row_q8_2_x4(q, y, D); ++ q += stride_q; ++ y += D/QK8_2; ++ } ++ } ++ ++ template ++ static inline void convert(int nq, int stride_q, const float * q, block_q8_KV * y) { ++ for (int i = 0; i < nq; ++i) { ++ quantize_row_q8_KV(q, y, D); ++ q += stride_q; ++ ++y; ++ } ++ } ++}; ++ ++template ++struct HelperQ80R8 : public BaseHelper { ++ using Base = BaseHelper; ++ constexpr static ggml_type type = GGML_TYPE_Q8_0_R8; ++#ifdef __AVX2__ ++ constexpr static int block_size_q = QK8_2; ++ using block_q8 = block_q8_2; ++#else ++ constexpr static int block_size_q = QK8_0; ++ using block_q8 = block_q8_0; ++#endif ++ HelperQ80R8(const char * data, int stride) : Base(data, stride) {} ++ HelperQ80R8(int nk, const HelperQ80& q8) : Base(q8.data, q8.stride) { ++ r4 = repack(nk, q8); ++ Base::data = (const char *)r4.data(); ++ Base::stride = (D/QK8_0)*sizeof(block_q8_0); ++ } ++ ++ static void repack(int nk, const char * q8_data, int q8_stride, block_q8_0_r8 * y) { ++ constexpr int nblock = D/QK8_0; ++ const block_q8_0 * x8[8]; ++#ifdef __ARM_NEON ++ int8x16x2_t m0, m1, m2, m3; ++#endif ++ for (int row = 0; row < nk; row += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q8_0 *)(q8_data + (row + k)*q8_stride); ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 8; ++k) y[ib].d[k] = x8[k][ib].d; ++#ifdef __AVX2__ ++ auto m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4][ib].qs), _mm_loadu_si128((const __m128i *)x8[0][ib].qs)); ++ auto m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5][ib].qs), _mm_loadu_si128((const __m128i *)x8[1][ib].qs)); ++ auto m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6][ib].qs), _mm_loadu_si128((const __m128i *)x8[2][ib].qs)); ++ auto m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7][ib].qs), _mm_loadu_si128((const __m128i *)x8[3][ib].qs)); ++ auto t0 = _mm256_unpacklo_epi32(m0, m1); ++ auto t1 = _mm256_unpacklo_epi32(m2, m3); ++ auto t2 = _mm256_unpackhi_epi32(m0, m1); ++ auto t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++//#ifdef HAVE_FANCY_SIMD ++// m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++// m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++// m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++// m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++//#endif ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 0, m0); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 1, m1); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 2, m2); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 3, m3); ++ m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[0][ib].qs+1)); ++ m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[1][ib].qs+1)); ++ m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[2][ib].qs+1)); ++ m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[3][ib].qs+1)); ++ t0 = _mm256_unpacklo_epi32(m0, m1); ++ t1 = _mm256_unpacklo_epi32(m2, m3); ++ t2 = _mm256_unpackhi_epi32(m0, m1); ++ t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++//#ifdef HAVE_FANCY_SIMD ++// m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++// m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++// m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++// m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++//#endif ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 4, m0); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 5, m1); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 6, m2); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 7, m3); ++#elif defined __ARM_NEON ++ for (int l = 0; l < 2; ++l) { ++ m0.val[0] = vld1q_s8(x8[0][ib].qs+16*l); m0.val[1] = vld1q_s8(x8[4][ib].qs+16*l); ++ m1.val[0] = vld1q_s8(x8[1][ib].qs+16*l); m1.val[1] = vld1q_s8(x8[5][ib].qs+16*l); ++ m2.val[0] = vld1q_s8(x8[2][ib].qs+16*l); m2.val[1] = vld1q_s8(x8[6][ib].qs+16*l); ++ m3.val[0] = vld1q_s8(x8[3][ib].qs+16*l); m3.val[1] = vld1q_s8(x8[7][ib].qs+16*l); ++ auto row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[0]), vreinterpretq_s32_s8(m1.val[0])); ++ auto row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[0]), vreinterpretq_s32_s8(m3.val[0])); ++ m0.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[1]), vreinterpretq_s32_s8(m1.val[1])); ++ row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[1]), vreinterpretq_s32_s8(m3.val[1])); ++ m0.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ vst1q_s8_x2(y[ib].qs + 0 + 128*l, m0); ++ vst1q_s8_x2(y[ib].qs + 32 + 128*l, m1); ++ vst1q_s8_x2(y[ib].qs + 64 + 128*l, m2); ++ vst1q_s8_x2(y[ib].qs + 96 + 128*l, m3); ++ } ++#else ++ for (int l = 0; l < 4; ++l) { ++ for (int k = 0; k < 8; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i+ 0] = x8[k][ib].qs[i+4*l+ 0]; ++ y[ib].qs[32*l+4*k+i+128] = x8[k][ib].qs[i+4*l+16]; ++ } ++ } ++#endif ++ } ++ y += nblock; ++ } ++ } ++ ++ static std::vector repack(int nk, const HelperQ80& q8) { ++ static_assert(D%QK8_0 == 0); ++ GGML_ASSERT(nk%8 == 0); ++ constexpr int nblock = D/QK8_0; ++ std::vector result(nblock * nk/8); ++ auto y = result.data(); ++ repack(nk, q8.data, q8.stride, y); ++ return result; ++ } ++ ++ std::vector r4; ++}; ++ ++// TODO: unite this with the above ++template ++struct HelperQ8KVR8 : public BaseHelper { ++ using Base = BaseHelper; ++ constexpr static ggml_type type = GGML_TYPE_Q8_KV_R8; ++ constexpr static int block_size_q = D; ++ using block_q8 = block_q8_KV; ++ ++ struct block_q8_KV_r8 { ++ float d[8]; ++ int8_t qs[8*D]; ++ }; ++ ++ HelperQ8KVR8(int nk, const HelperQ8KV& q8) : Base(q8.data, q8.stride) { ++ r4 = repack(nk, q8); ++ Base::data = (const char *)r4.data(); ++ Base::stride = sizeof(block_q8_KV_r8)/8; ++ } ++ ++ static std::vector repack(int nk, const HelperQ8KV& q8) { ++ static_assert(D%32 == 0); ++ GGML_ASSERT(nk%8 == 0); ++ std::vector result(nk/8); ++ auto y = result.data(); ++#ifdef __ARM_NEON ++ int8x16x2_t m0, m1, m2, m3; ++#endif ++ const int8_t * x8[8]; ++ for (int ix = 0; ix < nk/8; ++ix) { ++ for (int k = 0; k < 8; ++k) { ++ auto dptr = (const float *)(q8.data + (8*ix + k)*q8.stride); ++ y[ix].d[k] = dptr[0]; ++ x8[k] = (const int8_t *)(dptr + 2); ++ } ++ for (int ib = 0; ib < D/16; ++ib) { ++#ifdef __AVX2__ ++ auto m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4]+ib), _mm_loadu_si128((const __m128i *)x8[0]+ib)); ++ auto m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5]+ib), _mm_loadu_si128((const __m128i *)x8[1]+ib)); ++ auto m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6]+ib), _mm_loadu_si128((const __m128i *)x8[2]+ib)); ++ auto m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7]+ib), _mm_loadu_si128((const __m128i *)x8[3]+ib)); ++ auto t0 = _mm256_unpacklo_epi32(m0, m1); ++ auto t1 = _mm256_unpacklo_epi32(m2, m3); ++ auto t2 = _mm256_unpackhi_epi32(m0, m1); ++ auto t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++//#ifdef HAVE_FANCY_SIMD ++// m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++// m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++// m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++// m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++//#endif ++ _mm256_storeu_si256((__m256i *)y[ix].qs + 4*ib+0, m0); ++ _mm256_storeu_si256((__m256i *)y[ix].qs + 4*ib+1, m1); ++ _mm256_storeu_si256((__m256i *)y[ix].qs + 4*ib+2, m2); ++ _mm256_storeu_si256((__m256i *)y[ix].qs + 4*ib+3, m3); ++#elif defined __ARM_NEON ++ // TODO ++ m0.val[0] = vld1q_s8(x8[0]+16*ib); m0.val[1] = vld1q_s8(x8[4]+16*ib); ++ m1.val[0] = vld1q_s8(x8[1]+16*ib); m1.val[1] = vld1q_s8(x8[5]+16*ib); ++ m2.val[0] = vld1q_s8(x8[2]+16*ib); m2.val[1] = vld1q_s8(x8[6]+16*ib); ++ m3.val[0] = vld1q_s8(x8[3]+16*ib); m3.val[1] = vld1q_s8(x8[7]+16*ib); ++ auto row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[0]), vreinterpretq_s32_s8(m1.val[0])); ++ auto row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[0]), vreinterpretq_s32_s8(m3.val[0])); ++ m0.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[1]), vreinterpretq_s32_s8(m1.val[1])); ++ row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[1]), vreinterpretq_s32_s8(m3.val[1])); ++ m0.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ vst1q_s8_x2(y[ix].qs + 0 + 128*ib, m0); ++ vst1q_s8_x2(y[ix].qs + 32 + 128*ib, m1); ++ vst1q_s8_x2(y[ix].qs + 64 + 128*ib, m2); ++ vst1q_s8_x2(y[ix].qs + 96 + 128*ib, m3); ++#else ++ // TODO ++ for (int l = 0; l < 4; ++l) { ++ for (int k = 0; k < 8; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i+ 0] = x8[k][ib].qs[i+4*l+ 0]; ++ y[ib].qs[32*l+4*k+i+128] = x8[k][ib].qs[i+4*l+16]; ++ } ++ } ++#endif ++ } ++ } ++ return result; ++ } ++ ++ std::vector r4; ++}; ++ ++struct HelperQ40 final : public BaseHelper { ++ using Base = BaseHelper; ++ constexpr static ggml_type type = GGML_TYPE_Q4_0; ++#if defined __AVX2__ ++ using block_q8 = block_q8_2; ++ constexpr static int block_size_q = QK8_2; ++#else ++ using block_q8 = block_q8_0; ++ constexpr static int block_size_q = QK8_0; ++#endif ++ HelperQ40(const char * data, int stride) : Base(data, stride) {} ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ int j = F16::block_size*i; ++ auto dl = (const block_q4_0 *)Base::lblock(l1) + j/QK4_0; ++#ifdef __aarch64__ ++ auto vd = F16::set1(*(const float16_t *)&dl->d); ++ auto q = vld1q_u8(dl->qs); ++ q = j%QK4_0 ? vshrq_n_u8(q, 4) : vandq_u8(q, mask); ++ q = vaddq_s8(q, m8); ++ v1 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_low_s8(q)))); ++ v2 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_high_s8(q)))); ++#else ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++ auto q = _mm_loadu_si128((const __m128i *)dl->qs); ++#ifdef __AVX512F__ ++ auto ql = _mm_add_epi8(_mm_and_si128(q, mask), m8); ++ auto qh = _mm_add_epi8(_mm_and_si128(_mm_srli_epi16(q, 4), mask), m8); ++ v1 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(ql))); ++ v2 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(qh))); ++#else ++ if (j%QK4_0) q = _mm_srli_epi16(q, 4); ++ auto q16 = _mm256_cvtepi8_epi16(_mm_add_epi8(_mm_and_si128(q, mask), m8)); ++ v1 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16)))); ++ v2 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16, 1)))); ++#endif ++#endif ++ } ++ ++#ifdef __AVX2__ ++ const __m128i mask = _mm_set1_epi8(0xf); ++ const __m128i m8 = _mm_set1_epi8(-8); ++#else ++ const uint8x16_t mask = vdupq_n_u8(0xf); ++ const int8x16_t m8 = vdupq_n_s8(-8); ++#endif ++}; ++ ++struct HelperQ41 final : public BaseHelper { ++ using Base = BaseHelper; ++ using block_q8 = block_q8_2; ++ constexpr static ggml_type type = GGML_TYPE_Q4_1; ++ constexpr static int block_size_q = QK8_2; ++ HelperQ41(const char * data, int stride) : Base(data, stride) {} ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ int j = F16::block_size*i; ++ auto dl = (const block_q4_1 *)Base::lblock(l1) + j/QK4_1; ++#ifdef __aarch64__ ++ auto vd = F16::set1(*(const float16_t *)&dl->d); ++ auto vm = F16::set1(*(const float16_t *)&dl->m); ++ auto q = vld1q_u8(dl->qs); ++ q = (j%QK4_1) ? vshrq_n_u8(q, 4) : vandq_u8(q, mask); ++ v1 = vfmaq_f16(vm, vd, vcvtq_f16_u16(vmovl_u8(vget_low_u8(q)))); ++ v2 = vfmaq_f16(vm, vd, vcvtq_f16_u16(vmovl_u8(vget_high_u8(q)))); ++#else ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++ auto vm = F16::set1(GGML_FP16_TO_FP32(dl->m)); ++ auto q = _mm_loadu_si128((const __m128i *)dl->qs); ++#ifdef __AVX512F__ ++ auto ql = _mm_and_si128(q, mask); ++ auto qh = _mm_and_si128(_mm_srli_epi16(q, 4), mask); ++ v1 = _mm512_fmadd_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(ql)), vm); ++ v2 = _mm512_fmadd_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(qh)), vm); ++#else ++ if (j%QK4_1) q = _mm_srli_epi16(q, 4); ++ auto q16 = _mm256_cvtepi8_epi16(_mm_and_si128(q, mask)); ++ v1 = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16))), vm); ++ v2 = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16, 1))), vm); ++#endif ++#endif ++ } ++ ++#ifdef __aarch64__ ++ const uint8x16_t mask = vdupq_n_u8(0xf); ++#else ++ const __m128i mask = _mm_set1_epi8(0xf); ++#endif ++}; ++ ++struct HelperIQ4nl final : public BaseHelper { ++ using Base = BaseHelper; ++ constexpr static ggml_type type = GGML_TYPE_IQ4_NL; ++#ifdef __aarch64__ ++ using block_q8 = block_q8_0; ++ HelperIQ4nl(const char * data, int stride) : Base(data, stride), values(vld1q_s8(iq4k_values)) {} ++ constexpr static int block_size_q = QK8_0; ++#else ++ HelperIQ4nl(const char * data, int stride) : Base(data, stride) {} ++ using block_q8 = block_q8_2; ++ constexpr static int block_size_q = QK8_2; ++#endif ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ int j = F16::block_size*i; ++ auto dl = (const block_iq4_nl *)Base::lblock(l1) + j/QK4_0; ++#ifdef __aarch64__ ++ auto vd = F16::set1(*(const float16_t *)&dl->d); ++ auto q = vld1q_u8(dl->qs); ++ q = j%QK4_0 ? vshrq_n_u8(q, 4) : vandq_u8(q, mask); ++ q = vqtbl1q_s8(values, q); ++ v1 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_low_s8(q)))); ++ v2 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_high_s8(q)))); ++#else ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++ auto q = _mm_loadu_si128((const __m128i *)dl->qs); ++#ifdef __AVX512F__ ++ auto ql = _mm_shuffle_epi8(values, _mm_and_si128(q, mask)); ++ auto qh = _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(q, 4), mask)); ++ v1 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(ql))); ++ v2 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(qh))); ++#else ++ if (j%QK4_0) q = _mm_srli_epi16(q, 4); ++ auto q16 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(values, _mm_and_si128(q, mask))); ++ v1 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16)))); ++ v2 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16, 1)))); ++#endif ++#endif ++ } ++ ++#ifdef __aarch64__ ++ const uint8x16_t mask = vdupq_n_u8(0xf); ++ const int8x16_t values; ++#else ++ const __m128i mask = _mm_set1_epi8(0xf); ++ const __m128i values = _mm_loadu_si128((const __m128i *)iq4k_values); ++#endif ++}; ++ ++struct HelperQ60 final : public BaseHelper { ++ constexpr static ggml_type type = GGML_TYPE_Q6_0; ++#ifdef __aarch64__ ++ using block_q8 = block_q8_0; ++ constexpr static int block_size_q = QK8_0; ++#else ++ using block_q8 = block_q8_2; ++ constexpr static int block_size_q = QK8_2; ++#endif ++ using Base = BaseHelper; ++ HelperQ60(const char * data, int stride) : Base(data, stride) {} ++ ++ // Needed for v * softmax(k * q) ++ inline void load(int l1, int i, F16::Data& v1, F16::Data& v2) const { ++ int j = F16::block_size*i; ++ auto dl = (const block_q6_0 *)Base::lblock(l1) + j/QK6_0; ++#ifdef __aarch64__ ++ // TODO ++ const float16_t * d16 = (const float16_t *)&dl->d; ++ auto vd = F16::set1(d16[0]); ++ //auto vd = F16::set1(*(const float16_t *)&dl->d); ++ auto qh8 = vld1_u8(dl->qh); ++ auto qh = vcombine_u8(vshl_n_u8(qh8, 4), qh8); ++ auto qs = vld1q_u8(dl->qs); ++ qs = j%QK4_0 ? vshrq_n_u8(qs, 4) : vandq_u8(qs, mask_l); ++ qs = vorrq_u8(qs, vandq_u8(mask_h, j%QK4_0 ? vshrq_n_u8(qh, 2) : qh)); ++ qs = vaddq_s8(qs, m32); ++ v1 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_low_s8(qs)))); ++ v2 = vmulq_f16(vd, vcvtq_f16_s16(vmovl_s8(vget_high_s8(qs)))); ++#else ++ auto vd = F16::set1(GGML_FP16_TO_FP32(dl->d)); ++ auto bl = _mm_loadu_si128((const __m128i *)dl->qs); ++ uint64_t aux64; std::memcpy(&aux64, dl->qh, 8); ++ auto bh = _mm_set_epi64x(aux64, aux64 << 4); ++#ifdef __AVX512F__ ++ auto ql = _mm_add_epi8(_mm_or_si128(_mm_and_si128(bl, mask_l), _mm_and_si128(bh, mask_h)), m32); ++ auto qh = _mm_add_epi8(_mm_or_si128(_mm_and_si128(_mm_srli_epi16(bl, 4), mask_l), _mm_and_si128(_mm_srli_epi16(bh, 2), mask_h)), m32); ++ v1 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(ql))); ++ v2 = _mm512_mul_ps(vd, _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(qh))); ++#else ++ if (j%QK4_0) { ++ bl = _mm_srli_epi16(bl, 4); ++ bh = _mm_srli_epi16(bh, 2); ++ } ++ auto q16 = _mm256_cvtepi8_epi16(_mm_add_epi8(_mm_or_si128(_mm_and_si128(bl, mask_l), _mm_and_si128(bh, mask_h)), m32)); ++ v1 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16)))); ++ v2 = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16, 1)))); ++#endif ++#endif ++ } ++ ++#ifdef __AVX2__ ++ const __m128i mask_l = _mm_set1_epi8(0x0f); ++ const __m128i mask_h = _mm_set1_epi8(0x30); ++ const __m128i m32 = _mm_set1_epi8(-32); ++#else ++ const uint8x16_t mask_l = vdupq_n_u8(0x0f); ++ const uint8x16_t mask_h = vdupq_n_u8(0x30); ++ const int8x16_t m32 = vdupq_n_s8(-32); ++#endif ++}; ++ ++template ++struct FlashMS { ++ constexpr static int q_step = q_step_in; ++ constexpr static int k_step = k_step_in; ++// Something goes wrong when storing and manipulating K*Q as fp16. ++// It works for some models (e.g., Gemma-2), but not for others (e.g., LLaMA-3.1-8B). ++// As I wasn't able to find where we lose precision, let's comment this out ++// for now and do the K*Q part in fp32. ++//#ifdef __aarch64__ ++// using cache_t = float16_t; ++//#else ++// using cache_t = float; ++//#endif ++ using cache_t = float; ++ ++ FlashMS(float scale, float softcap) : vscale(F16::set1(scale)), softcap(softcap), h_inf(GGML_FP32_TO_FP16(-INFINITY)) {} ++ ++ inline void init_qstep() { ++ for (int j = 0; j < q_step; ++j) { ++ S[j] = 0; M[j] = -INFINITY; ++ } ++ } ++ ++ inline void update_M(int j, float smax) { ++ if (smax == -INFINITY) { ++ std::memset(cache + k_step*j, 0, k_step*sizeof(float)); ++ need_scaling[j] = M[j] == -INFINITY ? 2 : 0; ++ return; ++ } ++ need_scaling[j] = 0; ++ if (smax > M[j]) { ++ if (M[j] > -INFINITY) { ++ float m = expf(M[j] - smax); ++ vms[j] = m; ++ need_scaling[j] = 1; ++ S[j] *= m; ++ } else { ++ need_scaling[j] = 2; ++ S[j] = 0; ++ } ++ M[j] = smax; ++ } ++ } ++ ++#ifdef __aarch64__ ++ inline void update_S(int j, float32x4_t * vk) { ++ auto vm = vdupq_n_f32(M[j]); ++ auto vsum = vdupq_n_f32(0); ++ for (int l = 0; l < k_step/4; ++l) { ++ vk[l] = v_expf(vsubq_f32(vk[l], vm)); ++ vsum = vaddq_f32(vsum, vk[l]); ++ F16::store(cache + k_step*j + 4*l, vk[l]); ++ } ++ S[j] += vaddvq_f32(vsum); ++ } ++#else ++ inline void update_S(int j, F16::Data * vk) { ++ auto vm = F16::set1(M[j]); ++ for (int l = 0; l < k_step/F16::block_size; ++l) { ++ vk[l] = v_expf(F16::sub(vk[l], vm)); ++ F16::store(cache + k_step*j + F16::block_size*l, vk[l]); ++ } ++ S[j] += F16::reduce_add(vk); ++ } ++#endif ++ ++#ifdef __aarch64__ ++ inline float load_and_scale(int j, float32x4_t * vk) { ++ float32x4_t vmax = vdupq_n_f32(-INFINITY); ++ // Something goes wrong when storing and manipulating K*Q as fp16. ++ // It works for some models (e.g., Gemma-2), but not for others (e.g., LLaMA-3.1-8B). ++ // As I wasn't able to find where we lose precision, let's comment this out ++ // for now and do the K*Q part in fp32. ++ //if (softcap <= 0.0f) { ++ // for (int l = 0; l < k_step/F16::block_size; ++l) { ++ // auto val = F16::mul(vscale, F16::load(cache + k_step*j + F16::block_size*l)); ++ // vk[2*l+0] = vcvt_f32_f16(vget_low_f16(val)); ++ // vk[2*l+1] = vcvt_f32_f16(vget_high_f16(val)); ++ // vmax = vmaxq_f32(vmax, vmaxq_f32(vk[2*l+0], vk[2*l+1])); ++ // } ++ //} else { ++ // auto v_softcap = vdupq_n_f32(softcap); ++ // for (int l = 0; l < k_step/F16::block_size; ++l) { ++ // auto val = F16::mul(vscale, F16::load(cache + k_step*j + F16::block_size*l)); ++ // vk[2*l+0] = vcvt_f32_f16(vget_low_f16(val)); ++ // vk[2*l+1] = vcvt_f32_f16(vget_high_f16(val)); ++ // vk[2*l+0] = vmulq_f32(v_softcap, v_tanh(vk[2*l+0])); ++ // vk[2*l+1] = vmulq_f32(v_softcap, v_tanh(vk[2*l+1])); ++ // vmax = vmaxq_f32(vmax, vmaxq_f32(vk[2*l+0], vk[2*l+1])); ++ // } ++ //} ++ auto vscale32 = vcvt_f32_f16(vget_low_f16(vscale)); ++ if (softcap <= 0.0f) { ++ for (int l = 0; l < k_step/4; ++l) { ++ vk[l] = vmulq_f32(vscale32, vld1q_f32(cache + k_step*j + 4*l)); ++ vmax = vmaxq_f32(vmax, vk[l]); ++ } ++ } else { ++ auto v_softcap = vdupq_n_f32(softcap); ++ for (int l = 0; l < k_step/4; ++l) { ++ vk[l] = vmulq_f32(vscale32, vld1q_f32(cache + k_step*j + 4*l)); ++ vk[l] = vmulq_f32(v_softcap, v_tanh(vk[l])); ++ vmax = vmaxq_f32(vmax, vk[l]); ++ } ++ } ++ return vmaxvq_f32(vmax); ++ } ++ inline float load_apply_mask_and_scale(int j, float32x4_t * vk, const char * mask) { ++ auto vzero = vdupq_n_f16(0); ++ auto vinf = vdupq_n_f32(-INFINITY); ++ for (int l = 0; l < k_step/8; ++l) { ++ auto vm = vceqq_f16(vzero, vld1q_f16((const float16_t *)mask + 8*l)); ++ auto vm1 = vzip1q_u16(vm, vm); ++ auto vm2 = vzip2q_u16(vm, vm); ++ auto kq = vld1q_f32_x2(cache + k_step*j + 8*l); ++ vk[2*l+0] = vreinterpretq_f32_u32(vorrq_u32(vandq_u32(vreinterpretq_u32_f32(kq.val[0]), vm1), ++ vbicq_u32(vreinterpretq_u32_f32(vinf), vm1))); ++ vk[2*l+1] = vreinterpretq_f32_u32(vorrq_u32(vandq_u32(vreinterpretq_u32_f32(kq.val[1]), vm2), ++ vbicq_u32(vreinterpretq_u32_f32(vinf), vm2))); ++ } ++ float32x4_t vmax = vdupq_n_f32(-INFINITY); ++ auto vscale32 = vcvt_f32_f16(vget_low_f16(vscale)); ++ if (softcap <= 0.0f) { ++ for (int l = 0; l < k_step/4; ++l) { ++ vk[l] = vmulq_f32(vscale32, vk[l]); ++ vmax = vmaxq_f32(vmax, vk[l]); ++ } ++ } else { ++ auto v_softcap = vdupq_n_f32(softcap); ++ for (int l = 0; l < k_step/4; ++l) { ++ vk[l] = vmulq_f32(vscale32, vk[l]); ++ vk[l] = vmulq_f32(v_softcap, v_tanh(vk[l])); ++ vmax = vmaxq_f32(vmax, vk[l]); ++ } ++ } ++ return vmaxvq_f32(vmax); ++ } ++#else ++ inline float load_and_scale(int j, F16::Data * vk) { ++ if (softcap <= 0.0f) { ++ for (int l = 0; l < k_step/F16::block_size; ++l) vk[l] = F16::mul(vscale, F16::load(cache + k_step*j + F16::block_size*l)); ++ } else { ++ auto v_softcap = F16::set1(softcap); ++ for (int l = 0; l < k_step/F16::block_size; ++l) { ++ auto val = F16::load(cache + k_step*j + F16::block_size*l); ++ vk[l] = F16::mul(v_softcap, v_tanh(F16::mul(vscale, val))); ++ } ++ } ++ return F16::reduce_max(vk); ++ } ++ static inline __m256 apply_mask(int l, const char * mask, __m256 val, [[maybe_unused]] __m256 vinf) { ++ return _mm256_add_ps(val, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)mask+l))); ++ //auto m128 = _mm_loadu_si128((const __m128i *)mask+l); ++ //m128 = _mm_cmpeq_epi16(m128, _mm_setzero_si128()); ++ //auto m256 = _mm256_cvtepi16_epi32(m128); ++ //auto mf = _mm256_castsi256_ps(_mm256_or_si256(m256, _mm256_slli_epi32(m256, 16))); ++ //return _mm256_or_ps(_mm256_and_ps(mf, val), _mm256_andnot_ps(mf, vinf)); ++ } ++#ifdef __AVX512F__ ++ static inline __m512 apply_mask(int l, const char * mask, __m512 val, __m512 vinf) { ++ auto m256 = _mm256_loadu_si256((const __m256i *)mask+l); ++ m256 = _mm256_cmpeq_epi16(m256, _mm256_setzero_si256()); ++ auto m512 = _mm512_cvtepi16_epi32(m256); ++ auto mf = _mm512_castsi512_ps(_mm512_or_si512(m512, _mm512_slli_epi32(m512, 16))); ++ return _mm512_or_ps(_mm512_and_ps(mf, val), _mm512_andnot_ps(mf, vinf)); ++ } ++#endif ++ inline float load_apply_mask_and_scale(int j, F16::Data * vk, const char * mask) { ++#ifdef HAVE_FANCY_SIMD ++ auto vzero = _mm256_set1_epi16(0); ++ auto vinf = _mm512_set1_ps(-INFINITY); ++ if (softcap <= 0) { ++ for (int l = 0; l < k_step/F16::block_size; ++l) { ++ auto m16 = _mm256_cmpeq_epi16_mask(_mm256_loadu_si256((const __m256i *)mask + l), vzero); ++ vk[l] = _mm512_mask_mul_ps(vinf, m16, vscale, F16::load(cache + k_step*j + F16::block_size*l)); ++ } ++ } else { ++ auto v_softcap = F16::set1(softcap); ++ for (int l = 0; l < k_step/F16::block_size; ++l) { ++ auto m16 = _mm256_cmpeq_epi16_mask(_mm256_loadu_si256((const __m256i *)mask + l), vzero); ++ vk[l] = _mm512_mask_mul_ps(vinf, m16, v_softcap, v_tanh(F16::mul(vscale, F16::load(cache + k_step*j + F16::block_size*l)))); ++ } ++ } ++#else ++ auto vinf = F16::set1(-INFINITY); ++ for (int l = 0; l < k_step/F16::block_size; ++l) { ++ vk[l] = apply_mask(l, mask, F16::load(cache + k_step*j + F16::block_size*l), vinf); ++ } ++ if (softcap <= 0) { ++ for (int l = 0; l < k_step/F16::block_size; ++l) vk[l] = F16::mul(vscale, vk[l]); ++ } else { ++ auto v_softcap = F16::set1(softcap); ++ for (int l = 0; l < k_step/F16::block_size; ++l) vk[l] = F16::mul(v_softcap, v_tanh(F16::mul(vscale, vk[l]))); ++ } ++#endif ++ return F16::reduce_max(vk); ++ } ++#endif ++ ++#ifdef __aarch64__ ++ inline void update_M_S(int j, float32x4_t * vk) { ++ float smax = load_and_scale(j, vk); ++ update_M(j, smax); ++ if (M[j] > -INFINITY) update_S(j, vk); ++ } ++ inline void update_M_S(int j, float32x4_t * vk, const char * mask) { ++ float smax = load_apply_mask_and_scale(j, vk, mask); ++ update_M(j, smax); ++ if (M[j] > -INFINITY) update_S(j, vk); ++ } ++#else ++ inline void update_M_S(int j, F16::Data * vk) { ++ float smax = load_and_scale(j, vk); ++ update_M(j, smax); ++ if (M[j] > -INFINITY) update_S(j, vk); ++ } ++ inline void update_M_S(int j, F16::Data * vk, const char * mask) { ++ float smax = load_apply_mask_and_scale(j, vk, mask); ++ update_M(j, smax); ++ if (M[j] > -INFINITY) update_S(j, vk); ++ } ++#endif ++ ++ cache_t cache[q_step*k_step]; ++ float S[q_step], M[q_step]; ++ int need_scaling[q_step]; ++ float vms[q_step]; ++ const F16::Data vscale; ++ const float softcap; ++ const ggml_half h_inf; ++ ++}; ++ ++template ++struct FlashQKV { ++ ++#ifdef __aarch64__ ++ using qkv_cache_t = float16_t; ++#else ++ using qkv_cache_t = float; ++#endif ++ ++ template ++ inline void accumulate_qkv_1(const VHelper& vh, const FMS& fms) { ++ static_assert(q_step == FMS::q_step); ++ F16::Data vq[D/F16::block_size]; ++ if (fms.need_scaling[0] == 2) { ++ for (int i = 0; i < D/F16::block_size; ++i) vq[i] = F16::zero(); ++ } else { ++ for (int i = 0; i < D/F16::block_size; ++i) vq[i] = F16::load(qkv_cache + F16::block_size*i); ++ if (fms.need_scaling[0] == 1) { ++ auto vms = F16::set1(fms.vms[0]); ++ for (int i = 0; i < D/F16::block_size; ++i) vq[i] = F16::mul(vms, vq[i]); ++ } ++ } ++ F16::Data v0, v1; ++ for (int l = 0; l < k_step; l += 4) { ++ auto vs0 = F16::set1(fms.cache[l + 0]); ++ auto vs1 = F16::set1(fms.cache[l + 1]); ++ auto vs2 = F16::set1(fms.cache[l + 2]); ++ auto vs3 = F16::set1(fms.cache[l + 3]); ++ for (int i = 0; i < D/F16::block_size; i += 2) { ++ vh.load(l+0, i, v0, v1); ++ vq[i+0] = F16::fmadd(vq[i+0], v0, vs0); ++ vq[i+1] = F16::fmadd(vq[i+1], v1, vs0); ++ vh.load(l+1, i, v0, v1); ++ vq[i+0] = F16::fmadd(vq[i+0], v0, vs1); ++ vq[i+1] = F16::fmadd(vq[i+1], v1, vs1); ++ vh.load(l+2, i, v0, v1); ++ vq[i+0] = F16::fmadd(vq[i+0], v0, vs2); ++ vq[i+1] = F16::fmadd(vq[i+1], v1, vs2); ++ vh.load(l+3, i, v0, v1); ++ vq[i+0] = F16::fmadd(vq[i+0], v0, vs3); ++ vq[i+1] = F16::fmadd(vq[i+1], v1, vs3); ++ } ++ } ++ for (int i = 0; i < D/F16::block_size; ++i) F16::store(qkv_cache + F16::block_size*i, vq[i]); ++ } ++ ++ // This fails for head sizes of 80 and 112 as D/16 is odd, so we cannot do steps of 2 ++ // Hence, for now, we will not handle head sizes of 80 and 112 ++ template ++ inline void accumulate_qkv(const VHelper& vh, const FMS& fms) { ++ static_assert(q_step == FMS::q_step); ++ if constexpr (q_step == 1) { ++ accumulate_qkv_1(vh, fms); ++ return; ++ } ++ for (int j = 0; j < q_step; ++j) { ++ auto R = qkv_cache + D*j; ++ if (fms.need_scaling[j] == 2) { ++ std::memset(R, 0, D*sizeof(qkv_cache_t)); ++ } ++ else if (fms.need_scaling[j] == 1) { ++ auto vms = F16::set1(fms.vms[j]); ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ F16::store(R + F16::block_size*i, F16::mul(vms, F16::load(R + F16::block_size*i))); ++ } ++ } ++ } ++#ifdef __AVX512F__ ++ if constexpr ((D/F16::block_size)%4 == 0) { ++ F16::Data v[16]; ++ F16::Data vs[4]; ++ for (int i = 0; i < D/F16::block_size; i += 4) { ++ for (int l = 0; l < k_step; l += 4) { ++ for (int k = 0; k < 4; ++k) { ++ vh.load(l+k, i+0, v[4*k+0], v[4*k+1]); ++ vh.load(l+k, i+2, v[4*k+2], v[4*k+3]); ++ } ++ for (int j = 0; j < q_step; ++j) { ++ auto R = qkv_cache + D*j; ++ auto s1 = F16::load(R + F16::block_size*(i+0)); ++ auto s2 = F16::load(R + F16::block_size*(i+1)); ++ auto s3 = F16::load(R + F16::block_size*(i+2)); ++ auto s4 = F16::load(R + F16::block_size*(i+3)); ++ F16::set4(fms.cache + k_step*j + l, vs); ++ for (int k = 0; k < 4; ++k) { ++ s1 = F16::fmadd(s1, v[4*k+0], vs[k]); ++ s2 = F16::fmadd(s2, v[4*k+1], vs[k]); ++ s3 = F16::fmadd(s3, v[4*k+2], vs[k]); ++ s4 = F16::fmadd(s4, v[4*k+3], vs[k]); ++ } ++ F16::store(R + F16::block_size*(i+0), s1); ++ F16::store(R + F16::block_size*(i+1), s2); ++ F16::store(R + F16::block_size*(i+2), s3); ++ F16::store(R + F16::block_size*(i+3), s4); ++ } ++ } ++ } ++ return; ++ } ++#endif ++ F16::Data v[8]; ++#ifdef __AVX2__ ++ F16::Data vs[4]; ++#endif ++ for (int i = 0; i < D/F16::block_size; i += 2) { ++ for (int l = 0; l < k_step; l += 4) { ++ vh.load(l+0, i, v[0], v[4]); ++ vh.load(l+1, i, v[1], v[5]); ++ vh.load(l+2, i, v[2], v[6]); ++ vh.load(l+3, i, v[3], v[7]); ++ for (int j = 0; j < q_step; ++j) { ++ auto R = qkv_cache + D*j; ++ auto s1 = F16::load(R + F16::block_size*(i+0)); ++ auto s2 = F16::load(R + F16::block_size*(i+1)); ++#ifdef __AVX2__ ++ F16::set4(fms.cache + k_step*j + l, vs); ++ for (int k = 0; k < 4; ++k) { ++ s1 = F16::fmadd(s1, v[k+0], vs[k]); ++ s2 = F16::fmadd(s2, v[k+4], vs[k]); ++ } ++#else ++ auto vs = F16::set4(fms.cache + k_step*j + l); ++ s1 = F16::fmadd_lane0(s1, v[0], vs); ++ s2 = F16::fmadd_lane0(s2, v[4], vs); ++ s1 = F16::fmadd_lane1(s1, v[1], vs); ++ s2 = F16::fmadd_lane1(s2, v[5], vs); ++ s1 = F16::fmadd_lane2(s1, v[2], vs); ++ s2 = F16::fmadd_lane2(s2, v[6], vs); ++ s1 = F16::fmadd_lane3(s1, v[3], vs); ++ s2 = F16::fmadd_lane3(s2, v[7], vs); ++#endif ++ F16::store(R + F16::block_size*(i+0), s1); ++ F16::store(R + F16::block_size*(i+1), s2); ++ } ++ } ++ } ++ } ++ ++ template ++ inline void accumulate_qkv(int nq1, const VHelper& vh, const FMS& fms) { ++ static_assert(q_step == FMS::q_step); ++ if (nq1 == 1) { ++ accumulate_qkv_1(vh, fms); ++ return; ++ } ++ F16::Data v[8]; ++ for (int j = 0; j < nq1; ++j) { ++ auto R = qkv_cache + D*j; ++ if (fms.need_scaling[j] == 2) { ++ std::memset(R, 0, D*sizeof(qkv_cache_t)); ++ } ++ else if (fms.need_scaling[j] == 1) { ++ auto vms = F16::set1(fms.vms[j]); ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ F16::store(R + F16::block_size*i, F16::mul(vms, F16::load(R + F16::block_size*i))); ++ } ++ } ++ } ++ for (int i = 0; i < D/F16::block_size; i += 2) { ++ for (int l = 0; l < k_step; l += 4) { ++ vh.load(l+0, i, v[0], v[4]); ++ vh.load(l+1, i, v[1], v[5]); ++ vh.load(l+2, i, v[2], v[6]); ++ vh.load(l+3, i, v[3], v[7]); ++ for (int j = 0; j < nq1; ++j) { ++ auto R = qkv_cache + D*j; ++ auto s1 = F16::load(R + F16::block_size*(i+0)); ++ auto s2 = F16::load(R + F16::block_size*(i+1)); ++ auto vs = F16::set4(fms.cache + k_step*j + l); ++ s1 = F16::fmadd_lane0(s1, v[0], vs); ++ s2 = F16::fmadd_lane0(s2, v[4], vs); ++ s1 = F16::fmadd_lane1(s1, v[1], vs); ++ s2 = F16::fmadd_lane1(s2, v[5], vs); ++ s1 = F16::fmadd_lane2(s1, v[2], vs); ++ s2 = F16::fmadd_lane2(s2, v[6], vs); ++ s1 = F16::fmadd_lane3(s1, v[3], vs); ++ s2 = F16::fmadd_lane3(s2, v[7], vs); ++ F16::store(R + F16::block_size*(i+0), s1); ++ F16::store(R + F16::block_size*(i+1), s2); ++ } ++ } ++ } ++ } ++ ++ template ++ inline void normalize_and_store_1row(const FMS& fms, int j, qkv_cache_t * R, float * qkv, const float * sinkf) const { ++ static_assert(q_step == FMS::q_step); ++ float S = fms.S[j]; ++ if (sinkf) { ++ float s = *sinkf; ++ if (s > fms.M[j]) { ++ float m = expf(fms.M[j] - s); ++ auto vm = F16::set1(m); ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ auto Ri = R + F16::block_size*i; ++ F16::store(Ri, F16::mul(vm, F16::load(Ri))); ++ } ++ S = S*m + 1; ++ } else { ++ S += expf(s - fms.M[j]); ++ } ++ } ++ GGML_ASSERT(S > 0); ++ auto norm = F16::set1(1/S); ++ //auto norm = F16::set1(fms.S[j] > 0 ? 1/fms.S[j] : 0.f); ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ auto r = F16::load(R + F16::block_size*i); ++ F16::store(qkv + F16::block_size*i, F16::mul(norm, r)); ++ } ++ } ++ ++ template ++ inline void normalize_and_store(const FMS& fms, int nq1, int stride_qkv, float * qkv, const float * sinkf, float * M, float * S) { ++ static_assert(q_step == FMS::q_step); ++ if (M && S) { ++ std::memcpy(M, fms.M, nq1*sizeof(float)); ++ std::memcpy(S, fms.S, nq1*sizeof(float)); ++ auto R = qkv_cache; ++ for (int j = 0; j < nq1; ++j) { ++#ifdef __aarch64__ ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ F16::store(qkv + F16::block_size*i, F16::load(R + F16::block_size*i)); ++ } ++#else ++ std::memcpy(qkv, R, D*sizeof(float)); ++#endif ++ qkv += stride_qkv; ++ R += D; ++ } ++ } else { ++ auto R = qkv_cache; ++ for (int j = 0; j < nq1; ++j) { ++ normalize_and_store_1row(fms, j, R, qkv, sinkf); ++ qkv += stride_qkv; ++ R += D; ++ } ++ } ++ } ++ ++ template ++ inline void normalize_and_store(const FMS& fms, int stride_qkv, float * qkv, const float * sinkf, float * M, float * S) { ++ static_assert(q_step == FMS::q_step); ++ if (M && S) { ++ std::memcpy(M, fms.M, q_step*sizeof(float)); ++ std::memcpy(S, fms.S, q_step*sizeof(float)); ++ auto R = qkv_cache; ++ for (int j = 0; j < q_step; ++j) { ++#ifdef __aarch64__ ++ for (int i = 0; i < D/F16::block_size; ++i) { ++ F16::store(qkv + F16::block_size*i, F16::load(R + F16::block_size*i)); ++ } ++#else ++ std::memcpy(qkv, R, D*sizeof(float)); ++#endif ++ qkv += stride_qkv; ++ R += D; ++ } ++ } else { ++ auto R = qkv_cache; ++ for (int j = 0; j < q_step; ++j) { ++ normalize_and_store_1row(fms, j, R, qkv, sinkf); ++ qkv += stride_qkv; ++ R += D; ++ } ++ } ++ } ++ ++ // qkv_cache_t qkv_cache[D*q_step]; ++ // The initializer is not actually required. But the compiler cannot figure out that when qkv_cache is ++ // first used for q_step rows, fms.need_scaling[j] is always 2, which zeroes the content of qkv_cache. ++ // As a result, we get an infinite stream of warnings about uninitialized variable use (one for each ++ // combination of D, q_step, k_step), which is extremely annoying. Hence, I succumb to the trend of ++ // constantly being saved by others (the compiler in this case), and add this 100% unnecessary initialization. ++ qkv_cache_t qkv_cache[D*q_step]; // = {}; ++ //qkv_cache_t * qkv_cache; ++}; ++ ++template ++struct FlashQKfp32 { ++ static_assert(D%F16::block_size == 0 && D <= 576); ++ static_assert(k_step%F16::block_size == 0); ++ static_assert(q_step <= 4 || q_step%4 == 0); ++ ++ template ++ static inline void multiply_mask_kq(const KHelper& kh, int stride_q, int stride_m, const q_float * q, const char * mask, ++ FlashMS& fms) { ++#ifdef __AVX2__ ++ constexpr int nrc_k = 8; ++ static_assert(k_step%nrc_k == 0); ++#endif ++ DataInfo info{fms.cache, (const char *)q, k_step, stride_q*sizeof(q_float), 0, 1, nullptr}; ++ iqk_gemm_default_floats(D, q_step, kh.block, kh.stride, info, k_step); ++#ifdef __AVX2__ ++ F16::Data vk[k_step/F16::block_size]; ++#else ++ float32x4_t vk[k_step/4]; ++#endif ++ for (int j = 0; j < q_step; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++ } ++ ++ template ++ static inline void multiply_mask_kq(int nq, const KHelper& kh, int stride_q, int stride_m, const q_float * q, const char * mask, ++ FlashMS& fms) { ++#ifdef __AVX2__ ++ constexpr int nrc_k = 8; ++ static_assert(k_step%nrc_k == 0); ++#endif ++ DataInfo info{fms.cache, (const char *)q, k_step, stride_q*sizeof(q_float), 0, 1, nullptr}; ++ iqk_gemm_default_floats(D, nq, kh.block, kh.stride, info, k_step); ++#ifdef __AVX2__ ++ F16::Data vk[k_step/F16::block_size]; ++#else ++ float32x4_t vk[k_step/4]; ++#endif ++ for (int j = 0; j < nq; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++ } ++ ++#ifdef __aarch64__ ++ static inline void convert(int nq, int stride_q, const float * q, float16_t * q_f16) { ++ for (int i = 0; i < nq; ++i) { ++ for (int j = 0; j < D; j += 8) { ++ auto val1_f32 = vld1q_f32(q + j + 0); ++ auto val2_f32 = vld1q_f32(q + j + 4); ++ auto val_f16 = vcombine_f16(vcvt_f16_f32(val1_f32), vcvt_f16_f32(val2_f32)); ++ vst1q_f16(q_f16 + j, val_f16); ++ } ++ q += stride_q; ++ q_f16 += D; ++ } ++ } ++#endif ++ ++ template ++ static inline void mul_mask_kq(const KHelper& kh, int stride_m, ++ const block_q8 * q, const char * mask, FlashMS& fms) { ++ // As far as I can tell, this static assert is a remnant of the times where the matrix multiplications were done inline ++ // here with bespoke kernels instead of just using the regular mat mul kernels. But, just in case, leaving it in place ++ // but commneted out. ++ //constexpr int kMaxQ = 8; ++ //static_assert(q_step < kMaxQ || q_step%kMaxQ == 0); ++ DataInfo info{fms.cache, (const char *)q, k_step, (D/KHelper::block_size_q)*sizeof(block_q8), 0, 1, nullptr}; ++ if constexpr (std::is_same_v> || ++ std::is_same_v>) { ++ iqk_gemm_q8kv_fa(D, q_step, kh.type, kh.block, kh.stride, info, k_step); ++ } else { ++ iqk_gemm_legacy_fa(D, q_step, kh.type, kh.block, kh.stride, info, k_step); ++ } ++#ifdef __aarch64__ ++ float32x4_t vk[k_step/4]; ++ for (int j = 0; j < q_step; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++#else ++ F16::Data vk[k_step/F16::block_size]; ++ for (int j = 0; j < q_step; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++#endif ++ } ++ ++ template ++ static inline void mul_mask_kq(int nq, const KHelper& kh, int stride_m, ++ const block_q8 * q, const char * mask, FlashMS& fms) { ++ GGML_ASSERT(nq < q_step); ++ DataInfo info{fms.cache, (const char *)q, k_step, (D/KHelper::block_size_q)*sizeof(block_q8), 0, 1, nullptr}; ++ if constexpr (std::is_same_v> || ++ std::is_same_v>) { ++ iqk_gemm_q8kv_fa(D, nq, kh.type, kh.block, kh.stride, info, k_step); ++ } else { ++ iqk_gemm_legacy_fa(D, nq, kh.type, kh.block, kh.stride, info, k_step); ++ } ++#ifdef __aarch64__ ++ float32x4_t vk[k_step/4]; ++ for (int j = 0; j < nq; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++#else ++ F16::Data vk[k_step/F16::block_size]; ++ for (int j = 0; j < nq; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++#endif ++ } ++}; ++ ++template ++void compute_helper(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ FlashMS& fms, ++ FlashQKV& fqkv, ++ const float * q, const char * mask, float * qkv, ++ const float * sinkf, float * M, float * S) { ++#ifdef __aarch64__ ++ float16_t q_f16[Dk*q_step]; ++#endif ++ ++ for (int i1 = 0; i1 < nq1/q_step; ++i1) { ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++#ifdef __aarch64__ ++ KQHelper::convert(q_step, stride_q, q, q_f16); ++#endif ++ auto mr = mask; ++ int nk1_eff = mask_effective_nk1(mr, q_step, stride_m, nk1, k_step); ++ for (int k1 = 0; k1 < nk1_eff/k_step; ++k1) { ++#ifdef __aarch64__ ++ KQHelper::multiply_mask_kq(kh, Dk, stride_m, q_f16, mr, fms); ++#else ++ KQHelper::multiply_mask_kq(kh, stride_q, stride_m, q, mr, fms); ++#endif ++ fqkv.accumulate_qkv(vh, fms); ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++ fqkv.normalize_and_store(fms, stride_qkv, qkv, sinkf, M, S); ++ ++ q += q_step*stride_q; ++ mask += q_step*stride_m; ++ qkv += q_step*stride_qkv; ++ if (M && S) { M += q_step; S += q_step; } ++ } ++ int n_left = nq1 - q_step*(nq1/q_step); ++ if (n_left > 0) { ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++#ifdef __aarch64__ ++ KQHelper::convert(n_left, stride_q, q, q_f16); ++#endif ++ auto mr = mask; ++ for (int k1 = 0; k1 < nk1/k_step; ++k1) { ++#ifdef __aarch64__ ++ KQHelper::multiply_mask_kq(n_left, kh, Dk, stride_m, q_f16, mr, fms); ++#else ++ KQHelper::multiply_mask_kq(n_left, kh, stride_q, stride_m, q, mr, fms); ++#endif ++ fqkv.accumulate_qkv(n_left, vh, fms); ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++ fqkv.normalize_and_store(fms, n_left, stride_qkv, qkv, sinkf, M, S); ++ } ++} ++ ++template ++void compute_helper_q(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ FlashMS& fms, ++ FlashQKV& fqkv, ++ const float * q, const char * mask, float * qkv, ++ const float * sinkf, float * M, float * S, char * qptr) { ++ auto q8 = (typename KHelper::block_q8 *)qptr; ++ // This optimization fails under certain conditions (see https://github.com/ikawrakow/ik_llama.cpp/issues/1205) ++ // => disabling until I figure out what goes wrong ++ if constexpr (q_step >= 4 && std::is_same_v) { ++ if (nq1 == q_step) { ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++ block_q8_0_r8 q8r8[Dk/QK8_0 * k_step/8]; ++ HelperQ80R8 khr8((const char *)q8r8, Dk/QK8_0*sizeof(block_q8_0)); ++ auto q8r = (typename HelperQ80R8::block_q8 *)qptr; ++ HelperQ80::convert(q_step, stride_q, q, q8r); ++ auto mr = mask; ++ int nk1_eff = mask_effective_nk1(mr, q_step, stride_m, nk1, k_step); ++ for (int k1 = 0; k1 < nk1_eff/k_step; ++k1) { ++ HelperQ80R8::repack(k_step, kh.block, kh.stride, q8r8); ++ KQHelper::mul_mask_kq(khr8, stride_m, q8r, mr, fms); ++ fqkv.accumulate_qkv(vh, fms); ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++ fqkv.normalize_and_store(fms, stride_qkv, qkv, sinkf, M, S); ++ return; ++ } ++ } ++#if FA_TIMING ++ Perf perf(false); ++#endif ++ for (int i1 = 0; i1 < nq1/q_step; ++i1) { ++#if FA_TIMING ++ auto t1 = Perf::cur_time(); ++#endif ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++ HelperQ80::convert(q_step, stride_q, q, q8); ++#if FA_TIMING ++ perf.accum_nolock(0, t1); ++#endif ++ auto mr = mask; ++ int nk1_eff = mask_effective_nk1(mr, q_step, stride_m, nk1, k_step); ++ for (int k1 = 0; k1 < nk1_eff/k_step; ++k1) { ++#if FA_TIMING ++ t1 = Perf::cur_time(); ++ KQHelper::mul_mask_kq(kh, stride_m, q8, mr, fms); ++ perf.accum_nolock(1, t1); ++ t1 = Perf::cur_time(); ++ fqkv.accumulate_qkv(vh, fms); ++ perf.accum_nolock(2, t1); ++#else ++ KQHelper::mul_mask_kq(kh, stride_m, q8, mr, fms); ++ fqkv.accumulate_qkv(vh, fms); ++#endif ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++#if FA_TIMING ++ t1 = Perf::cur_time(); ++ fqkv.normalize_and_store(fms, stride_qkv, qkv, sinkf, M, S); ++ perf.accum_nolock(3, t1); ++#else ++ fqkv.normalize_and_store(fms, stride_qkv, qkv, sinkf, M, S); ++#endif ++ ++ q += q_step*stride_q; ++ mask += q_step*stride_m; ++ qkv += q_step*stride_qkv; ++ if (M && S) { M += q_step; S += q_step; } ++ } ++ int n_left = nq1 - q_step*(nq1/q_step); ++ if (n_left > 0) { ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++ HelperQ80::convert(n_left, stride_q, q, q8); ++ auto mr = mask; ++ for (int k1 = 0; k1 < nk1/k_step; ++k1) { ++ KQHelper::mul_mask_kq(n_left, kh, stride_m, q8, mr, fms); ++ fqkv.accumulate_qkv(n_left, vh, fms); ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++ fqkv.normalize_and_store(fms, n_left, stride_qkv, qkv, sinkf, M, S); ++ } ++#if FA_TIMING ++ Perf::instance().add(perf); ++#endif ++} ++ ++char * get_q_storage(size_t size) { ++ thread_local std::vector q_storage; ++ if (q_storage.size() < size) q_storage.resize(size); ++ return q_storage.data(); ++} ++ ++// Some of the methods in FlashAttn have two identical implementations that only differ by ++// one version using a loop over the template parameter q_step, while the other using a loop ++// over an input parameter nq (these are loops over the rows of q^T). I dislike this a lot, ++// but performance drops signficantly if I remove the version with fixed q_step iterations. ++// We only instantiate FlashAttn with q_step = 1 and q_step = 4 or 8 (depending on head size D), ++// so when we have to process Nq rows, we process q_step*(Nq/q_step) using fixed q_step loops, ++// and use the variable nq version (with lower performance) only for the remaining i1...q_step-1 ++// rows (if Nq is not a multiple of q_step). One could have made the number of q^T rows to ++// process template parameter of such functions, but this would result in the compiler generating ++// q_step-1 versions of these functions for us, which I though was too much with q_step = 8. ++template ++struct FlashAttn { ++ static_assert(Dk%F16::block_size == 0 && Dk <= 576); ++ static_assert(Dv%F16::block_size == 0 && Dv <= 512); ++ static_assert(k_step%F16::block_size == 0); ++ static_assert(q_step <= 4 || q_step%4 == 0); ++ ++ FlashAttn(float scale, float softcap, const float * sinkf) : fms(scale, softcap), sinkf(sinkf) {} ++ ++ template ++ void compute(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ const float * q, const char * mask, float * qkv, [[maybe_unused]] float * M, [[maybe_unused]] float * S) { ++ if constexpr (std::is_same_v || ++ std::is_same_v || ++ std::is_same_v || ++ std::is_same_v || ++ std::is_same_v> || ++ std::is_same_v || ++ std::is_same_v> || ++ std::is_same_v>) { ++ constexpr size_t kMaxOnStackSize = 576; ++ //auto q_size = q_step*(Dk/KHelper::block_size_q)*sizeof(typename KHelper::block_q8); ++ auto q_size = q_step*(Dk/QK8_2*sizeof(block_q8_2)); ++ q_size = GGML_PAD(q_size, 64); ++ if (q_size > kMaxOnStackSize) { ++ auto qptr = get_q_storage(q_size); ++ if (false && nq1 >= 8) { ++ if constexpr (std::is_same_v) { ++#if FA_TIMING ++ auto t1 = Perf::cur_time(); ++ HelperQ80R8 khr4(nk1, kh); ++ Perf::instance().accum(4, t1); ++#else ++ HelperQ80R8 khr4(nk1, kh); ++#endif ++ compute_helper_q, VHelper, FlashQKfp32>( ++ khr4, vh, nq1, nk1, stride_q, stride_m, stride_qkv, fms, fqkv, q, mask, qkv, sinkf, M, S, qptr); ++ return; ++ ++ } ++#if GGML_IQK_FA_ALL_QUANTS ++ if constexpr (std::is_same_v>) { ++#if FA_TIMING ++ auto t1 = Perf::cur_time(); ++ HelperQ8KVR8 khr4(nk1, kh); ++ Perf::instance().accum(4, t1); ++#else ++ HelperQ8KVR8 khr4(nk1, kh); ++#endif ++ compute_helper_q, VHelper, FlashQKfp32>( ++ khr4, vh, nq1, nk1, stride_q, stride_m, stride_qkv, fms, fqkv, q, mask, qkv, sinkf, M, S, qptr); ++ return; ++ } ++#endif ++ } ++ compute_helper_q>( ++ kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, fms, fqkv, q, mask, qkv, sinkf, M, S, qptr); ++ ++ } ++ else { ++ typename KHelper::block_q8 q8[q_step*(Dk/KHelper::block_size_q)]; ++ compute_helper_q>( ++ kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, fms, fqkv, q, mask, qkv, sinkf, M, S, (char *)q8); ++ } ++ } ++ else { ++ compute_helper>( ++ kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, fms, fqkv, q, mask, qkv, sinkf, M, S); ++ } ++ } ++ ++ FlashMS fms; ++ FlashQKV fqkv; ++ const float * sinkf; ++ ++}; ++ ++#ifdef __AVX512BF16__ ++ ++template ++struct HelperBF16 final : public BaseHelper { ++ using Base = BaseHelper; ++ HelperBF16(const char * data, int stride) : Base(data, stride) {} ++ inline void load(int l1, __m512bh * vk) const { ++ auto dr = Base::lblock(l1); ++ for (int i = 0; i < D/32; ++i) vk[i] = __m512bh(_mm512_loadu_si512((const __m512i*)dr + i)); ++ } ++ ++ inline void load(int l1, int i, __m512& v1, __m512& v2) const { ++ auto dr = Base::lblock(l1); ++ v1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)dr + i + 0)), 16)); ++ v2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)dr + i + 1)), 16)); ++ } ++ ++ inline void load_2(int l1, __m512bh * vk) const { ++ load(l1+0, vk+0); ++ load(l1+1, vk+D/32); ++ } ++ ++ inline void load_4(int l1, __m512bh * vk) const { ++ load(l1+0, vk+0); ++ load(l1+1, vk+1*D/32); ++ load(l1+2, vk+2*D/32); ++ load(l1+3, vk+3*D/32); ++ } ++ ++ inline void load_8(int l1, __m512bh * vk) const { ++ for (int k = 0; k < 8; ++k) load(l1 + k, vk + k*D/32); ++ } ++}; ++ ++template ++struct FlashQKbf16 { ++ //static_assert(D%32 == 0 && D <= 256); ++ static_assert(D%32 == 0 && D <= 576); ++ static_assert(k_step%32 == 0); ++ static_assert(q_step <= 4 || q_step%4 == 0); ++ ++ static inline void mult_mask_kq_one(int l1, int m1, int stride_q, int stride_m, const float * q, const char * mask, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ // q index is q_step*i1 + m1 ++ // k index is k_step*k1 + l1 ++ const ggml_half * mp = (const ggml_half *)(mask + stride_m*m1); ++ fms.cache[k_step*m1 + l1 + 0] = fms.cache[k_step*m1 + l1 + 1] = -INFINITY; ++ if (mp[l1+0] == fms.h_inf && mp[l1+1] == fms.h_inf) { ++ return; ++ } ++ auto qr = q + m1*stride_q; ++ for (int i = 0; i < D/32; ++i) { ++ auto val1 = _mm512_loadu_ps(qr + 32*i); ++ auto val2 = _mm512_loadu_ps(qr + 32*i + 16); ++ qv[i] = _mm512_cvtne2ps_pbh(val2, val1); ++ } ++ if (mp[l1+0] != fms.h_inf) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i], qv[i]); ++ fms.cache[k_step*m1 + l1 + 0] = _mm512_reduce_add_ps(vsum); ++ } ++ if (mp[l1+1] != fms.h_inf) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+D/32], qv[i]); ++ fms.cache[k_step*m1 + l1 + 1] = _mm512_reduce_add_ps(vsum); ++ } ++ } ++ ++ static inline void mult_mask_kq_one(int l1, int m1, int stride_m, const ggml_bf16_t * q, const char * mask, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ // q index is q_step*i1 + m1 ++ // k index is k_step*k1 + l1 ++ const ggml_half * mp = (const ggml_half *)(mask + stride_m*m1); ++ fms.cache[k_step*m1 + l1 + 0] = fms.cache[k_step*m1 + l1 + 1] = -INFINITY; ++ if (mp[l1+0] == fms.h_inf && mp[l1+1] == fms.h_inf) { ++ return; ++ } ++ auto qr = q + m1*D; ++ for (int i = 0; i < D/32; ++i) qv[i] = __m512bh(_mm512_loadu_si512((const __m512i*)qr + i)); ++ if (mp[l1+0] != fms.h_inf) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i], qv[i]); ++ fms.cache[k_step*m1 + l1 + 0] = _mm512_reduce_add_ps(vsum); ++ } ++ if (mp[l1+1] != fms.h_inf) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+D/32], qv[i]); ++ fms.cache[k_step*m1 + l1 + 1] = _mm512_reduce_add_ps(vsum); ++ } ++ } ++ ++ static inline void mult_mask_kq_4(int l1, int m1, int stride_q, int stride_m, const float * q, const char * mask, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ // q index is q_step*i1 + m1 ++ // k index is k_step*k1 + l1 ++ const ggml_half * mp = (const ggml_half *)(mask + stride_m*m1); ++ fms.cache[k_step*m1 + l1 + 0] = fms.cache[k_step*m1 + l1 + 1] = ++ fms.cache[k_step*m1 + l1 + 2] = fms.cache[k_step*m1 + l1 + 3] = -INFINITY; ++ if (mp[l1+0] == fms.h_inf && mp[l1+1] == fms.h_inf && mp[l1+2] == fms.h_inf && mp[l1+3] == fms.h_inf) { ++ return; ++ } ++ auto qr = q + m1*stride_q; ++ for (int i = 0; i < D/32; ++i) { ++ auto val1 = _mm512_loadu_ps(qr + 32*i); ++ auto val2 = _mm512_loadu_ps(qr + 32*i + 16); ++ qv[i] = _mm512_cvtne2ps_pbh(val2, val1); ++ } ++ for (int k = 0; k < 4; ++k) { ++ if (mp[l1+k] == fms.h_inf) continue; ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+k*(D/32)], qv[i]); ++ fms.cache[k_step*m1 + l1 + k] = _mm512_reduce_add_ps(vsum); ++ } ++ } ++ ++ static inline void mult_mask_kq_4(int l1, int m1, int stride_m, const ggml_bf16_t * q, const char * mask, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ // q index is q_step*i1 + m1 ++ // k index is k_step*k1 + l1 ++ const ggml_half * mp = (const ggml_half *)(mask + stride_m*m1); ++ fms.cache[k_step*m1 + l1 + 0] = fms.cache[k_step*m1 + l1 + 1] = ++ fms.cache[k_step*m1 + l1 + 2] = fms.cache[k_step*m1 + l1 + 3] = -INFINITY; ++ if (mp[l1+0] == fms.h_inf && mp[l1+1] == fms.h_inf && mp[l1+2] == fms.h_inf && mp[l1+3] == fms.h_inf) { ++ return; ++ } ++ auto qr = q + m1*D; ++ for (int i = 0; i < D/32; ++i) qv[i] = __m512bh(_mm512_loadu_si512((const __m512i *)qr + i)); ++ for (int k = 0; k < 4; ++k) { ++ if (mp[l1+k] == fms.h_inf) continue; ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+k*(D/32)], qv[i]); ++ fms.cache[k_step*m1 + l1 + k] = _mm512_reduce_add_ps(vsum); ++ } ++ } ++ ++ static inline __m128 hsum_float_4x4(__m128 * a) { ++ for (int i = 0; i < 2; ++i) a[i] = _mm_add_ps(_mm_unpacklo_ps(a[i], a[i+2]), _mm_unpackhi_ps(a[i], a[i+2])); ++ return _mm_add_ps(_mm_unpacklo_ps(a[0], a[1]), _mm_unpackhi_ps(a[0], a[1])); ++ } ++ ++ template ++ static inline void multiply_mask_kq(const KHelper& kh, int stride_q, int stride_m, const float * q, ++ const char * mask, FlashMS& fms) { ++ { ++ __m512bh qv[D/32]; ++ if constexpr (D <= 128) { ++ __m512bh vkh[D/8]; ++ for (int l1 = 0; l1 < k_step; l1 += 4) { ++ kh.load_4(l1, vkh); ++ for (int j = 0; j < q_step; ++j) { ++ mult_mask_kq_4(l1, j, stride_q, stride_m, q, mask, qv, vkh, fms); ++ } ++ } ++ } else { ++ __m512bh vkh[D/16]; ++ for (int l1 = 0; l1 < k_step; l1 += 2) { ++ kh.load_2(l1, vkh); ++ for (int j = 0; j < q_step; ++j) { ++ mult_mask_kq_one(l1, j, stride_q, stride_m, q, mask, qv, vkh, fms); ++ } ++ } ++ } ++ } ++ __m512 vk[k_step/16]; ++ for (int j = 0; j < q_step; ++j) { ++ fms.update_M_S(j, vk); ++ } ++ } ++ ++ static inline void mult_mask_kq_4(int l1, int m1, const ggml_bf16_t * q, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ auto qr = q + m1*D; ++ for (int i = 0; i < D/32; ++i) qv[i] = __m512bh(_mm512_loadu_si512((const __m512i *)qr + i)); ++ __m128 sum[4]; ++ for (int k = 0; k < 4; ++k) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+k*(D/32)], qv[i]); ++ auto aux = _mm256_add_ps(_mm512_castps512_ps256(vsum), _mm512_extractf32x8_ps(vsum, 1)); ++ sum[k] = _mm_add_ps(_mm256_castps256_ps128(aux), _mm256_extractf128_ps(aux, 1)); ++ } ++ //auto sum4 = _mm_mask_blend_ps(m8, hsum_float_4x4(sum), _mm_set1_ps(-INFINITY)); ++ //_mm_storeu_ps(fms.cache + k_step*m1 + l1, sum4); ++ _mm_storeu_ps(fms.cache + k_step*m1 + l1, hsum_float_4x4(sum)); ++ } ++ ++ static IQK_ALWAYS_INLINE __m256 hsum_float_8x8(__m256 * accm) { ++ for (int i = 0; i < 4; ++i) { ++ accm[i] = _mm256_add_ps(_mm256_permute2f128_ps(accm[i], accm[i+4], 0x20), _mm256_permute2f128_ps(accm[i], accm[i+4], 0x31)); ++ //accm[i] = _mm256_set_m128(_mm_add_ps(_mm256_castps256_ps128(accm[i+4]), _mm256_extractf128_ps(accm[i+4], 1)), ++ // _mm_add_ps(_mm256_castps256_ps128(accm[i+0]), _mm256_extractf128_ps(accm[i+0], 1))); ++ } ++ for (int i = 0; i < 2; ++i) accm[i] = _mm256_add_ps(_mm256_unpacklo_ps(accm[i], accm[i+2]), _mm256_unpackhi_ps(accm[i], accm[i+2])); ++ return _mm256_add_ps(_mm256_unpacklo_ps(accm[0], accm[1]), _mm256_unpackhi_ps(accm[0], accm[1])); ++ } ++ ++ static inline void mult_mask_kq_8(int l1, int m1, const ggml_bf16_t * q, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ auto qr = q + m1*D; ++ for (int i = 0; i < D/32; ++i) qv[i] = __m512bh(_mm512_loadu_si512((const __m512i *)qr + i)); ++ __m256 sum[8]; ++ for (int k = 0; k < 8; ++k) { ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+k*(D/32)], qv[i]); ++ sum[k] = _mm256_add_ps(_mm512_castps512_ps256(vsum), _mm512_extractf32x8_ps(vsum, 1)); ++ } ++ _mm256_storeu_ps(fms.cache + k_step*m1 + l1, hsum_float_8x8(sum)); ++ } ++ ++ static inline void mult_mask_kq_one(int l1, int m1, const ggml_bf16_t * q, ++ __m512bh * qv, const __m512bh * vkh, FlashMS& fms) { ++ auto qr = q + m1*D; ++ for (int i = 0; i < D/32; ++i) qv[i] = __m512bh(_mm512_loadu_si512((const __m512i*)qr + i)); ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i], qv[i]); ++ fms.cache[k_step*m1 + l1 + 0] = _mm512_reduce_add_ps(vsum); ++ vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vkh[i+D/32], qv[i]); ++ fms.cache[k_step*m1 + l1 + 1] = _mm512_reduce_add_ps(vsum); ++ } ++ ++#if FA_TIMING ++ template ++ static inline void multiply_mask_kq(const KHelper& kh, int stride_m, const ggml_bf16_t * q, ++ const char * mask, FlashMS& fms, Perf& perf) { ++ auto t1 = Perf::cur_time(); ++#else ++ template ++ static inline void multiply_mask_kq(const KHelper& kh, int stride_m, const ggml_bf16_t * q, ++ const char * mask, FlashMS& fms) { ++#endif ++ if constexpr (q_step == 1) { ++ __m512bh vq[D/32]; ++ __m512bh vk[D/32]; ++ __m256 sum[8]; ++ for (int i = 0; i < D/32; ++i) vq[i] = __m512bh(_mm512_loadu_si512((const __m512i *)q + i)); ++ for (int l = 0; l < k_step; l += 8) { ++ for (int k = 0; k < 8; ++k) { ++ kh.load(l+k, vk); ++ auto vsum = _mm512_setzero_ps(); ++ for (int i = 0; i < D/32; ++i) vsum = _mm512_dpbf16_ps(vsum, vk[i], vq[i]); ++ sum[k] = _mm256_add_ps(_mm512_castps512_ps256(vsum), _mm512_extractf32x8_ps(vsum, 1)); ++ } ++ _mm256_storeu_ps(fms.cache + l, hsum_float_8x8(sum)); ++ } ++ } ++ else { ++ __m512bh qv[D/32]; ++ if constexpr (D <= 128) { ++ __m512bh vkh[D/4]; ++ for (int l1 = 0; l1 < k_step; l1 += 8) { ++ kh.load_8(l1, vkh); ++ for (int j = 0; j < q_step; ++j) mult_mask_kq_8(l1, j, q, qv, vkh, fms); ++ } ++ } else { ++ __m512bh vkh[D/16]; ++ for (int l1 = 0; l1 < k_step; l1 += 2) { ++ kh.load_2(l1, vkh); ++ for (int j = 0; j < q_step; ++j) mult_mask_kq_one(l1, j, q, qv, vkh, fms); ++ } ++ } ++ } ++#if FA_TIMING ++ perf.accum_nolock(1, t1); ++ t1 = Perf::cur_time(); ++#endif ++ F16::Data vk[k_step/16]; ++ for (int j = 0; j < q_step; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++#if FA_TIMING ++ perf.accum_nolock(2, t1); ++#endif ++ } ++ ++ template ++ static inline void multiply_mask_kq(int nq, const KHelper& kh, int stride_m, const ggml_bf16_t * q, ++ const char * mask, FlashMS& fms) { ++ { ++ __m512bh qv[D/32]; ++ if constexpr (D <= 128) { ++ __m512bh vkh[D/8]; ++ for (int l1 = 0; l1 < k_step; l1 += 4) { ++ kh.load_4(l1, vkh); ++ for (int j = 0; j < nq; ++j) mult_mask_kq_4(l1, j, q, qv, vkh, fms); ++ } ++ } else { ++ __m512bh vkh[D/16]; ++ for (int l1 = 0; l1 < k_step; l1 += 2) { ++ kh.load_2(l1, vkh); ++ for (int j = 0; j < nq; ++j) mult_mask_kq_one(l1, j, q, qv, vkh, fms); ++ } ++ } ++ } ++ F16::Data vk[k_step/16]; ++ for (int j = 0; j < nq; ++j) { ++ fms.update_M_S(j, vk, mask + stride_m*j); ++ } ++ } ++ ++ template ++ static inline void multiply_mask_kq(int nq, const KHelper& kh, int stride_q, int stride_m, const float * q, ++ const char * mask, FlashMS& fms) { ++ { ++ __m512bh qv[D/32]; ++ __m512bh vkh[D/16]; ++ for (int l1 = 0; l1 < k_step; l1 += 2) { ++ kh.load_2(l1, vkh); ++ for (int m1 = 0; m1 < nq; ++m1) { ++ mult_mask_kq_one(l1, m1, stride_q, stride_m, q, mask, qv, vkh, fms); ++ } ++ } ++ } ++ __m512 vk[k_step/16]; ++ for (int j = 0; j < nq; ++j) { ++ fms.update_M_S(j, vk); ++ } ++ } ++ ++ static inline void convert(int stride_q, const float * q, ggml_bf16_t * bf16) { ++ auto qr = q; ++ for (int j = 0; j < q_step; ++j) { ++ for (int i = 0; i < D/32; ++i) { ++ auto val1 = _mm512_loadu_ps(qr + 32*i); ++ auto val2 = _mm512_loadu_ps(qr + 32*i + 16); ++ _mm512_storeu_si512((__m512i *)bf16 + i, (__m512i)_mm512_cvtne2ps_pbh(val2, val1)); ++ } ++ qr += stride_q; ++ bf16 += D; ++ } ++ } ++ ++ static inline void convert(int nq, int stride_q, const float * q, ggml_bf16_t * bf16) { ++ auto qr = q; ++ for (int j = 0; j < nq; ++j) { ++ for (int i = 0; i < D/32; ++i) { ++ auto val1 = _mm512_loadu_ps(qr + 32*i); ++ auto val2 = _mm512_loadu_ps(qr + 32*i + 16); ++ _mm512_storeu_si512((__m512i *)bf16 + i, (__m512i)_mm512_cvtne2ps_pbh(val2, val1)); ++ } ++ qr += stride_q; ++ bf16 += D; ++ } ++ } ++}; ++ ++template ++struct FlashAttnBF16 { ++ //static_assert(Dk%32 == 0 && Dk <= 256); ++ //static_assert(Dv%32 == 0 && Dv <= 256); ++ static_assert(Dk%32 == 0 && Dk <= 576); ++ static_assert(Dv%32 == 0 && Dv <= 512); ++ static_assert(k_step%32 == 0); ++ static_assert(q_step <= 4 || q_step%4 == 0); ++ ++ FlashAttnBF16(float scale, float softcap, const float * sinkf) : fms(scale, softcap), sinkf(sinkf) {} ++ ++ template ++ void compute(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ const float * q, const char * mask, float * qkv, [[maybe_unused]] float * M, [[maybe_unused]] float * S) { ++ ggml_bf16_t q_bf16[q_step*Dk]; ++#if FA_TIMING ++ Perf perf(false); ++#endif ++ for (int i1 = 0; i1 < nq1/q_step; ++i1) { ++#if FA_TIMING ++ auto t1 = Perf::cur_time(); ++#endif ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++ FlashQKbf16::convert(stride_q, q, q_bf16); ++#if FA_TIMING ++ perf.accum_nolock(0, t1); ++#endif ++ auto mr = mask; ++ int nk1_eff = mask_effective_nk1(mr, q_step, stride_m, nk1, k_step); ++ for (int k1 = 0; k1 < nk1_eff/k_step; ++k1) { ++#if FA_TIMING ++ //t1 = Perf::cur_time(); ++ FlashQKbf16::multiply_mask_kq(kh, stride_m, q_bf16, mr, fms, perf); ++ //perf.accum_nolock(1, t1); ++ t1 = Perf::cur_time(); ++ fqkv.accumulate_qkv(vh, fms); ++ perf.accum_nolock(3, t1); ++#else ++ FlashQKbf16::multiply_mask_kq(kh, stride_m, q_bf16, mr, fms); ++ fqkv.accumulate_qkv(vh, fms); ++#endif ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++#if FA_TIMING ++ t1 = Perf::cur_time(); ++#endif ++ fqkv.normalize_and_store(fms, stride_qkv, qkv, sinkf, M, S); ++#if FA_TIMING ++ perf.accum_nolock(4, t1); ++#endif ++ ++ q += q_step*stride_q; ++ mask += q_step*stride_m; ++ qkv += q_step*stride_qkv; ++ if (M && S) { M += q_step; S += q_step; } ++ } ++ int n_left = nq1 - q_step*(nq1/q_step); ++ if (n_left > 0) { ++ fms.init_qstep(); ++ kh.reset_block(); ++ vh.reset_block(); ++ FlashQKbf16::convert(n_left, stride_q, q, q_bf16); ++ auto mr = mask; ++ for (int k1 = 0; k1 < nk1/k_step; ++k1) { ++ FlashQKbf16::multiply_mask_kq(n_left, kh, stride_m, q_bf16, mr, fms); ++ fqkv.accumulate_qkv(n_left, vh, fms); ++ kh.next_block(k_step); ++ vh.next_block(k_step); ++ mr += k_step*sizeof(ggml_half); ++ } ++ fqkv.normalize_and_store(fms, n_left, stride_qkv, qkv, sinkf, M, S); ++ } ++#if FA_TIMING ++ Perf::instance().add(perf); ++#endif ++ } ++ ++ FlashMS fms; ++ FlashQKV fqkv; ++ const float * sinkf; ++}; ++#endif ++ ++template ++inline void iqk_flash_helper(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, int stride_m, int stride_qkv, ++ const float * q, const char * mask, float scale, float softcap, float * qkv, ++ const float * sinkf, float * M, float * S) { ++ ++ auto update = [&nq1, &mask, &q, &qkv, &M, &S, stride_q, stride_m, stride_qkv] (int n) { ++ nq1 -= n; ++ if (nq1 == 0) return true; ++ q += n*stride_q; ++ mask += n*stride_m; ++ qkv += n*stride_qkv; ++ if (M && S) { M += n; S += n; } ++ return false; ++ }; ++ if (nk1 >= 512) { ++ if (nq1 >= 128) { ++ int n_step = nq1/128; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 128*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(128*n_step)) return; ++ } ++ if (nq1 >= 64) { ++ int n_step = nq1/64; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 64*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(64*n_step)) return; ++ } ++ if (nq1 >= 32) { ++ int n_step = nq1/32; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 32*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(32*n_step)) return; ++ } ++ if (nq1 >= 16) { ++ int n_step = nq1/16; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 16*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(16*n_step)) return; ++ } ++ } ++ if (nq1 == 12) { ++ // Special case: TG for GLM-4.5/4.6 ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 12, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ return; ++ } ++ if (nq1 >= 8) { ++ int n_step = nq1/8; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 8*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(8*n_step)) return; ++ } ++ else if (nq1 >= 4) { ++ int n_step = nq1/4; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 4*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(4*n_step)) return; ++ } ++ else if (nq1 >= 2) { ++ int n_step = nq1/2; ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, 2*n_step, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ if (update(2*n_step)) return; ++ } ++ FlashAttn fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++} ++ ++#ifdef __AVX512BF16__ ++template ++inline void iqk_flash_helper_T(int nq1, int nk1, int stride_q, int stride_k, int stride_v, int stride_m, int stride_qkv, ++ const float * q, const char * k, const char * v, const char * mask, ++ float scale, float softcap, float * qkv, const float * sinkf, float * M, float * S) { ++ HelperBF16 kh(k, stride_k); ++ HelperBF16 vh(v, stride_v); ++ if (nk1 >= 4096) { ++ if (nq1 >= 64) { ++ FlashAttnBF16 fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ return; ++ } ++ else if (nq1 >= 16) { ++ FlashAttnBF16 fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ return; ++ } ++ } ++ if (nq1 >= 8) { ++ FlashAttnBF16 fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } else { ++ FlashAttnBF16 fa(scale, softcap, sinkf); ++ fa.compute(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, (const char *)mask, qkv, M, S); ++ } ++} ++#endif ++ ++template ++inline bool iqk_flash_helper_T(KHelper& kh, ggml_type type_v, ++ int nq1, int nk1, int stride_q, int stride_v, int stride_m, int stride_qkv, ++ const float * q, const char * v, const char * mask, ++ float scale, float softcap, float * qkv, const float * sinkf, float * M, float * S) { ++ ++ switch (type_v) { ++ case GGML_TYPE_F16: { ++ HelperF16 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#ifdef __AVX512BF16__ ++ case GGML_TYPE_BF16: { ++ HelperBF16 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#endif ++ case GGML_TYPE_Q8_0: { ++ HelperQ80 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q8_KV: { ++ HelperQ8KV vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q6_0: { ++ HelperQ60 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#if GGML_IQK_FA_ALL_QUANTS ++ case GGML_TYPE_Q4_0: { ++ HelperQ40 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q4_1: { ++ HelperQ41 vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_IQ4_NL: { ++ HelperIQ4nl vh(v, stride_v); ++ iqk_flash_helper(kh, vh, nq1, nk1, stride_q, stride_m, stride_qkv, q, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#endif ++ default: return false; ++ } ++ return true; ++} ++ ++template ++inline bool iqk_flash_helper_T(ggml_type type_k, ggml_type type_v, ++ int nq1, int nk1, int stride_q, int stride_k, int stride_v, int stride_m, int stride_qkv, ++ const float * q, const char * k, const char * v, const char * mask, ++ float scale, float softcap, float * qkv, const float * sinkf, float * M, float * S) { ++ ++ bool result = false; ++ switch (type_k) { ++ case GGML_TYPE_F16: { ++ HelperF16 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q8_0: { ++ HelperQ80 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q8_0_R8: { ++ HelperQ80R8 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q6_0: { ++ HelperQ60 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#if GGML_IQK_FA_ALL_QUANTS ++ case GGML_TYPE_Q8_KV: { ++ HelperQ8KV kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q4_0: { ++ HelperQ40 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_Q4_1: { ++ HelperQ41 kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++ case GGML_TYPE_IQ4_NL: { ++ HelperIQ4nl kh(k, stride_k); ++ result = iqk_flash_helper_T(kh, type_v, nq1, nk1, stride_q, stride_v, stride_m, stride_qkv, q, v, mask, scale, softcap, qkv, sinkf, M, S); ++ } break; ++#endif ++ default: break; ++ } ++ ++ return result; ++} ++ ++} ++ ++#define IQK_FA_CASE(name) bool name(int int_type_k, int int_type_v,int nq,int nk,\ ++ int stride_q, int stride_k, int stride_v, int stride_m, int stride_qkv,\ ++ const float * q, const void * k, const void * v, const void * mask,\ ++ float scale, float softcap,\ ++ float * qkv, const float * sinkf, float * M, float * S) ++ ++IQK_FA_CASE(iqk_fa_576_512); ++IQK_FA_CASE(iqk_fa_512_512); ++IQK_FA_CASE(iqk_fa_320_256); ++IQK_FA_CASE(iqk_fa_192_128); ++IQK_FA_CASE(iqk_fa_192_192); ++IQK_FA_CASE(iqk_fa_256_256); ++IQK_FA_CASE(iqk_fa_128_128); ++IQK_FA_CASE(iqk_fa_96_96); ++IQK_FA_CASE(iqk_fa_64_64); ++ ++#endif ++ +diff --git a/llama.cpp/ggml/src/iqk/ggml-common.h b/llama.cpp/ggml/src/iqk/ggml-common.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/ggml-common.h +@@ -0,0 +1,13 @@ ++// opencoti F5-opt W2 (#290) — iqk-private ggml-common.h shim (HARD ISOLATION). ++// ++// iqk TUs resolve `#include "ggml-common.h"` to THIS file (their own dir wins ++// over -Iggml/src). It forwards to llamafile's real, re-includable ggml-common.h ++// (so shared/GGUF-frozen types stay SINGLE-SOURCED — they reach every TU through ++// ggml-impl.h too, which includes the real header relative to ggml/src/), then ++// layers on ONLY the block types ik_llama.cpp adds (iqk_common_extra.h). ++// ++// NO #pragma once: the upstream header is intentionally re-includable with ++// different GGML_COMMON_DECL_*/IMPL_* each pass. The delta carries its own guard. ++#include "../ggml-common.h" ++#include "iqk_ggml_type_ext.h" // ik_llama ggml_type enum delta ++#include "iqk_common_extra.h" +diff --git a/llama.cpp/ggml/src/iqk/iqk_common.h b/llama.cpp/ggml/src/iqk/iqk_common.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_common.h +@@ -0,0 +1,958 @@ ++// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*- ++// vi: set et ft=cpp fenc=utf-8 :vi ++// ++// ++// Copyright (C) 2024 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++ ++#include "iqk_config.h" ++ ++#if defined IQK_IMPLEMENT ++ ++#include ++#include ++#include ++#include ++ ++#include "ggml-impl.h" ++#include "ggml-quants.h" ++#include "iqk_mul_mat.h" ++#include "iqk_quantize.h" ++ ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++ ++#define FA_TIMING 0 ++ ++#include ++#include ++#if FA_TIMING ++#include ++#include ++struct Perf { ++ using TimePoint = std::chrono::time_point; ++ std::array times = {}; ++ std::mutex mutex; ++ bool report; ++ static auto cur_time() { return std::chrono::high_resolution_clock::now(); } ++ inline void accum(int what, const TimePoint& t1) { ++ auto t2 = cur_time(); ++ auto dt = delta(t1, t2); ++ std::lock_guard lock(mutex); ++ times[what] += dt; ++ } ++ inline void accum_nolock(int what, const TimePoint& t1) { ++ auto t2 = cur_time(); ++ auto dt = delta(t1, t2); ++ times[what] += dt; ++ } ++ inline void add(const Perf& other) { ++ std::lock_guard lock(mutex); ++ for (int i = 0; i < int(times.size()); ++i) times[i] += other.times[i]; ++ } ++ Perf(bool r) : report(r) {} ++ ~Perf() { ++ if (report) { ++ double tot = 0; ++ for (auto& t : times) tot += t; ++ if (!tot) return; ++ printf("======================= Timing: %g ms in total\n", tot); ++ for (int i = 0; i < int(times.size()); ++i) { ++ if (times[i]) { ++ printf("%d: %g ms -> %g%c\n", i, times[i], 100*times[i]/tot, '%'); ++ } ++ } ++ } ++ } ++ static Perf& instance() { ++ static Perf p(true); ++ return p; ++ } ++ static double delta(const TimePoint& t1, const TimePoint& t2) { ++ return 1e-6*std::chrono::duration_cast(t2-t1).count(); ++ } ++}; ++#endif ++ ++#ifdef __AVX2__ ++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) ++#endif ++ ++typedef struct { ++ int32_t i1; ++ int32_t i2; ++} mmid_row_mapping; ++ ++struct DataInfo { ++ float * s; ++ const char * cy; ++ size_t bs; ++ size_t by; ++ int cur_y = 0; ++ int ne11; ++ const mmid_row_mapping * row_mapping = nullptr; ++ size_t bs2 = 0; ++ ++ inline const char * src1_row(int iy) const { ++ if (!row_mapping) return cy + (cur_y + iy)*by; ++ int i11 = row_mapping[cur_y + iy].i1 % ne11; ++ int i12 = row_mapping[cur_y + iy].i2; ++ return cy + (i11 + i12*ne11)*by; ++ } ++ ++ inline void store(int ix, int iy, float result) const { ++ *(dst_row(iy) + ix) = result; ++ } ++#ifdef __AVX__ ++ inline void store(int ix, int iy, __m128 result) const { ++ _mm_storeu_ps(dst_row(iy) + ix, result); ++ } ++ inline void store(int ix, int iy, __m256 result) const { ++ _mm256_storeu_ps(dst_row(iy) + ix, result); ++ } ++#endif ++#ifdef __AVX512F__ ++ inline void store(int ix, int iy, __m512 result) const { ++ _mm512_storeu_ps(dst_row(iy) + ix, result); ++ } ++#endif ++#ifdef __ARM_NEON ++ inline void store(int ix, int iy, float32x4_t result) const { ++ vst1q_f32(dst_row(iy) + ix, result); ++ } ++#endif ++ inline float * dst_row(int iy) const { ++ if (!row_mapping) return s + (cur_y + iy)*bs; ++ int i12 = row_mapping[cur_y + iy].i2; ++ int i1 = row_mapping[cur_y + iy].i1; ++ int i2 = i12; ++ return s + i1*bs + i2*bs2; ++ } ++}; ++ ++typedef void (*mul_mat_t)(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x); ++ ++#define IQK_MAX_NY 8 ++ ++#define IQK_SET_MUL_MAT_FUNCTIONS_T(kernel, Dequantizer, funcs) \ ++ funcs[0] = kernel;\ ++ funcs[1] = kernel;\ ++ funcs[2] = kernel;\ ++ funcs[3] = kernel;\ ++ funcs[4] = kernel;\ ++ funcs[5] = kernel;\ ++ funcs[6] = kernel;\ ++ funcs[7] = kernel;\ ++ ++#define IQK_SET_MUL_MAT_FUNCTIONS_T2(kernel, Dequantizer, Block, funcs) \ ++ funcs[0] = kernel;\ ++ funcs[1] = kernel;\ ++ funcs[2] = kernel;\ ++ funcs[3] = kernel;\ ++ funcs[4] = kernel;\ ++ funcs[5] = kernel;\ ++ funcs[6] = kernel;\ ++ funcs[7] = kernel;\ ++ ++#define IQK_SET_MUL_MAT_FUNCTIONS(kernel, funcs) \ ++ funcs[0] = kernel<1>;\ ++ funcs[1] = kernel<2>;\ ++ funcs[2] = kernel<3>;\ ++ funcs[3] = kernel<4>;\ ++ funcs[4] = kernel<5>;\ ++ funcs[5] = kernel<6>;\ ++ funcs[6] = kernel<7>;\ ++ funcs[7] = kernel<8>;\ ++ ++ ++// ================================================================================================== ++ ++static inline void make_q4_scales(const uint8_t * scales8, uint32_t * aux32) { ++ const uint16_t * scales = (const uint16_t *)scales8; ++ const uint32_t a0 = scales[0] | (scales[1] << 16); ++ const uint32_t a1 = scales[2] | (scales[3] << 16); ++ const uint32_t a2 = scales[4] | (scales[5] << 16); ++ aux32[3] = ((a2 >> 4) & 0x0f0f0f0f) | ((a1 >> 2) & 0x30303030); ++ aux32[1] = ((a2 >> 0) & 0x0f0f0f0f) | ((a0 >> 2) & 0x30303030); ++ aux32[2] = a1 & 0x3f3f3f3f; ++ aux32[0] = a0 & 0x3f3f3f3f; ++} ++ ++const uint64_t keven_signs[128] = { ++ 0x0101010101010101, 0xff010101010101ff, 0xff0101010101ff01, 0x010101010101ffff, ++ 0xff01010101ff0101, 0x0101010101ff01ff, 0x0101010101ffff01, 0xff01010101ffffff, ++ 0xff010101ff010101, 0x01010101ff0101ff, 0x01010101ff01ff01, 0xff010101ff01ffff, ++ 0x01010101ffff0101, 0xff010101ffff01ff, 0xff010101ffffff01, 0x01010101ffffffff, ++ 0xff0101ff01010101, 0x010101ff010101ff, 0x010101ff0101ff01, 0xff0101ff0101ffff, ++ 0x010101ff01ff0101, 0xff0101ff01ff01ff, 0xff0101ff01ffff01, 0x010101ff01ffffff, ++ 0x010101ffff010101, 0xff0101ffff0101ff, 0xff0101ffff01ff01, 0x010101ffff01ffff, ++ 0xff0101ffffff0101, 0x010101ffffff01ff, 0x010101ffffffff01, 0xff0101ffffffffff, ++ 0xff01ff0101010101, 0x0101ff01010101ff, 0x0101ff010101ff01, 0xff01ff010101ffff, ++ 0x0101ff0101ff0101, 0xff01ff0101ff01ff, 0xff01ff0101ffff01, 0x0101ff0101ffffff, ++ 0x0101ff01ff010101, 0xff01ff01ff0101ff, 0xff01ff01ff01ff01, 0x0101ff01ff01ffff, ++ 0xff01ff01ffff0101, 0x0101ff01ffff01ff, 0x0101ff01ffffff01, 0xff01ff01ffffffff, ++ 0x0101ffff01010101, 0xff01ffff010101ff, 0xff01ffff0101ff01, 0x0101ffff0101ffff, ++ 0xff01ffff01ff0101, 0x0101ffff01ff01ff, 0x0101ffff01ffff01, 0xff01ffff01ffffff, ++ 0xff01ffffff010101, 0x0101ffffff0101ff, 0x0101ffffff01ff01, 0xff01ffffff01ffff, ++ 0x0101ffffffff0101, 0xff01ffffffff01ff, 0xff01ffffffffff01, 0x0101ffffffffffff, ++ 0xffff010101010101, 0x01ff0101010101ff, 0x01ff01010101ff01, 0xffff01010101ffff, ++ 0x01ff010101ff0101, 0xffff010101ff01ff, 0xffff010101ffff01, 0x01ff010101ffffff, ++ 0x01ff0101ff010101, 0xffff0101ff0101ff, 0xffff0101ff01ff01, 0x01ff0101ff01ffff, ++ 0xffff0101ffff0101, 0x01ff0101ffff01ff, 0x01ff0101ffffff01, 0xffff0101ffffffff, ++ 0x01ff01ff01010101, 0xffff01ff010101ff, 0xffff01ff0101ff01, 0x01ff01ff0101ffff, ++ 0xffff01ff01ff0101, 0x01ff01ff01ff01ff, 0x01ff01ff01ffff01, 0xffff01ff01ffffff, ++ 0xffff01ffff010101, 0x01ff01ffff0101ff, 0x01ff01ffff01ff01, 0xffff01ffff01ffff, ++ 0x01ff01ffffff0101, 0xffff01ffffff01ff, 0xffff01ffffffff01, 0x01ff01ffffffffff, ++ 0x01ffff0101010101, 0xffffff01010101ff, 0xffffff010101ff01, 0x01ffff010101ffff, ++ 0xffffff0101ff0101, 0x01ffff0101ff01ff, 0x01ffff0101ffff01, 0xffffff0101ffffff, ++ 0xffffff01ff010101, 0x01ffff01ff0101ff, 0x01ffff01ff01ff01, 0xffffff01ff01ffff, ++ 0x01ffff01ffff0101, 0xffffff01ffff01ff, 0xffffff01ffffff01, 0x01ffff01ffffffff, ++ 0xffffffff01010101, 0x01ffffff010101ff, 0x01ffffff0101ff01, 0xffffffff0101ffff, ++ 0x01ffffff01ff0101, 0xffffffff01ff01ff, 0xffffffff01ffff01, 0x01ffffff01ffffff, ++ 0x01ffffffff010101, 0xffffffffff0101ff, 0xffffffffff01ff01, 0x01ffffffff01ffff, ++ 0xffffffffffff0101, 0x01ffffffffff01ff, 0x01ffffffffffff01, 0xffffffffffffffff, ++}; ++ ++#ifdef __AVX2__ ++ ++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) ++ ++static inline float hsum_float_4(__m128 x) { ++ x = _mm_add_ps(x, _mm_movehl_ps(x, x)); ++ x = _mm_add_ss(x, _mm_movehdup_ps(x)); ++ return _mm_cvtss_f32(x); ++} ++static inline float hsum_float_8(__m256 x) { ++ return hsum_float_4(_mm_add_ps(_mm256_castps256_ps128(x), _mm256_extractf128_ps(x, 1))); ++} ++static inline int hsum_i32_8(const __m256i a) { ++ const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1)); ++ const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128); ++ const __m128i sum64 = _mm_add_epi32(hi64, sum128); ++ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1)); ++ return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32)); ++} ++static inline float hmax_f32_8(__m256 x) { ++ __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)); ++ max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); ++ max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4)); ++ return _mm_cvtss_f32(max4); ++} ++static inline float hmax_float_8(__m256 x) { ++ __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)); ++ max4 = _mm_max_ps( max4, _mm_movehl_ps(max4, max4)); ++ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4)); ++ return _mm_cvtss_f32(max4); ++} ++static inline float hmin_float_8(__m256 x) { ++ __m128 min4 = _mm_min_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)); ++ min4 = _mm_min_ps( min4, _mm_movehl_ps(min4, min4)); ++ min4 = _mm_min_ss( min4, _mm_movehdup_ps( min4)); ++ return _mm_cvtss_f32(min4); ++} ++ ++static inline __m128 hsum_float_4x4(__m128 * accm) { ++ accm[0] = _mm_add_ps(_mm_unpacklo_ps(accm[0], accm[2]), _mm_unpackhi_ps(accm[0], accm[2])); ++ accm[1] = _mm_add_ps(_mm_unpacklo_ps(accm[1], accm[3]), _mm_unpackhi_ps(accm[1], accm[3])); ++ return _mm_add_ps(_mm_unpacklo_ps(accm[0], accm[1]), _mm_unpackhi_ps(accm[0], accm[1])); ++} ++static inline __m256 hsum_float_8x8(__m256 * accm) { ++ for (int i = 0; i < 4; ++i) { ++ accm[i] = _mm256_add_ps(_mm256_permute2f128_ps(accm[i], accm[i + 4], 0x20), _mm256_permute2f128_ps(accm[i], accm[i + 4], 0x31)); ++ //accm[i] = _mm256_set_m128(_mm_add_ps(_mm256_castps256_ps128(accm[i+4]), _mm256_extractf128_ps(accm[i+4], 1)), ++ // _mm_add_ps(_mm256_castps256_ps128(accm[i+0]), _mm256_extractf128_ps(accm[i+0], 1))); ++ } ++ for (int i = 0; i < 2; ++i) accm[i] = _mm256_add_ps(_mm256_unpacklo_ps(accm[i], accm[i + 2]), _mm256_unpackhi_ps(accm[i], accm[i + 2])); ++ return _mm256_add_ps(_mm256_unpacklo_ps(accm[0], accm[1]), _mm256_unpackhi_ps(accm[0], accm[1])); ++} ++static inline __m256 hsum_float_4x8(__m256 * accm) { ++ for (int i = 0; i < 2; ++i) accm[i] = _mm256_add_ps(_mm256_unpacklo_ps(accm[i], accm[i + 2]), _mm256_unpackhi_ps(accm[i], accm[i + 2])); ++ return _mm256_add_ps(_mm256_unpacklo_ps(accm[0], accm[1]), _mm256_unpackhi_ps(accm[0], accm[1])); ++} ++ ++static inline __m128i load_iq4nl_values_128() { ++ static const uint8_t kvalues_iq4nl[16] = {1, 24, 45, 63, 79, 93, 106, 118, 129, 141, 153, 166, 181, 197, 217, 241}; ++ return _mm_loadu_si128((const __m128i *)kvalues_iq4nl); ++} ++ ++static inline __m256i load_iq4nl_values_256() { ++ auto val128 = load_iq4nl_values_128(); ++ return MM256_SET_M128I(val128, val128); ++} ++ ++#ifdef HAVE_FANCY_SIMD ++static inline __m512i load_iq4nl_values_512() { ++ auto val256 = load_iq4nl_values_256(); ++ return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); ++} ++#endif ++ ++static inline __m128i load_iq4k_values_128() { ++ return _mm_loadu_si128((const __m128i *)iq4k_values); ++} ++ ++static inline __m256i load_iq4k_values_256() { ++ auto val128 = load_iq4k_values_128(); ++ return MM256_SET_M128I(val128, val128); ++} ++ ++template struct Q8 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q8(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8 *)info.src1_row(iy); ++ } ++ ++#ifdef HAVE_FANCY_SIMD ++ inline __m512i load_quants64(int iy, int i, int j) const { return _mm512_loadu_si512((const __m512i*)y[iy][i].qs + j); } ++#endif ++ inline __m256i load_quants(int iy, int i, int j) const { return _mm256_loadu_si256((const __m256i*)y[iy][i].qs + j); } ++ inline __m256i load_bsums(int iy, int i) const { return _mm256_loadu_si256((const __m256i*)y[iy][i].bsums); } ++ inline float scale(int iy, int i) const { return y[iy][i].d; } ++ ++ const block_q8 * y[nrc_y]; ++}; ++ ++template struct Q8_16 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q8_16(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto ptr = (const float *)info.src1_row(iy); ++ std::memcpy(d + 5*iy, ptr, 5*sizeof(float)); ++ y[iy] = (const int8_t *)(ptr + 5); ++ } ++ } ++ ++#ifdef HAVE_FANCY_SIMD ++ inline __m512i load_quants64(int iy, int i) const { return _mm512_loadu_si512((const __m512i*)y[iy] + i); } ++#endif ++ inline __m256i load_quants(int iy, int i) const { return _mm256_loadu_si256((const __m256i*)y[iy] + i); } ++ inline float scale(int iy, int k) const { return d[5*iy+k]; } ++ inline float sum_row(int iy) const { return d[5*iy + 4]; } ++ inline __m128 scale(int iy) const { return _mm_loadu_ps(d + 5*iy); } ++ ++ float d[5*nrc_y]; ++ const int8_t * y[nrc_y]; ++}; ++ ++struct Scales8KBase { ++ template ++ inline void accum_mins(const __m128i& mins128, const Q8& q8, int i, float c, __m256 * accd) const { ++ const __m256i mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, shuffles[1]), _mm_shuffle_epi8(mins128, shuffles[0])); ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i q8s = q8.load_bsums(iy, i); ++ const __m256i prod = _mm256_madd_epi16(mins, q8s); ++ accd[iy] = _mm256_fmadd_ps(_mm256_set1_ps(c*q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accd[iy]); ++ } ++ } ++ inline __m256i shuffle(__m128i mins) const { ++ return MM256_SET_M128I(_mm_shuffle_epi8(mins, shuffles[1]), _mm_shuffle_epi8(mins, shuffles[0])); ++ } ++ const __m128i shuffles[2] = {_mm_set_epi32(0x07060706, 0x05040504, 0x03020302, 0x01000100), ++ _mm_set_epi32(0x0f0e0f0e, 0x0d0c0d0c, 0x0b0a0b0a, 0x09080908)}; ++}; ++ ++template ++struct BaseDequantizer { ++ BaseDequantizer(const void * vx, size_t bx) : vx(vx), bx(bx) {} ++ inline void new_row(int ix) { ++ if constexpr (per_row_scale) { ++ if constexpr (is_f16) { ++ const ggml_half * dptr = (const ggml_half *)((const char *)vx + bx*ix); ++ d = GGML_FP16_TO_FP32(*dptr); ++ x = (const Block *)(dptr + 1); ++ } else { ++ const float * dptr = (const float *)((const char *)vx + bx*ix); ++ d = *dptr; ++ x = (const Block *)(dptr + 1); ++ } ++ } else { ++ x = (const Block *)((const char *)vx + bx*ix); ++ } ++ } ++ ++ const void * vx; ++ const size_t bx; ++ const Block * x; ++ ++ float d; ++}; ++ ++template ++static inline void multiply_add(const Bits& bits, const __m256i * scales, int j, int i, const Q8& q8, __m256i * sumi) { ++ if (j == 0) { ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ sumi[iy] = _mm256_dpwssd_epi32(_mm256_setzero_si256(), scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 0))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 1))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 2))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 3))); ++ } ++#else ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 0))); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 1))); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 2))); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 3))); ++ sumi[iy] = _mm256_add_epi32(_mm256_add_epi32(p1, p3), _mm256_add_epi32(p2, p4)); ++ } ++#endif ++ } else { ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 5))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 6))); ++ sumi[iy] = _mm256_dpwssd_epi32(sumi[iy], scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 7))); ++ } ++#else ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i p1 = _mm256_madd_epi16(scales[0], _mm256_maddubs_epi16(bits.values[0], q8.load_quants(iy, i, 4))); ++ const __m256i p2 = _mm256_madd_epi16(scales[1], _mm256_maddubs_epi16(bits.values[1], q8.load_quants(iy, i, 5))); ++ const __m256i p3 = _mm256_madd_epi16(scales[2], _mm256_maddubs_epi16(bits.values[2], q8.load_quants(iy, i, 6))); ++ const __m256i p4 = _mm256_madd_epi16(scales[3], _mm256_maddubs_epi16(bits.values[3], q8.load_quants(iy, i, 7))); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p1, p3)); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p2, p4)); ++ } ++#endif ++ } ++} ++ ++template ++static inline void multiply_add_avx2(const Bits& bits, const __m256i * scales, int j, int i, const Q8& q8, __m256i * sumi) { ++ __m256i p[4]; ++ if (j == 0) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ for (int k = 0; k < 4; ++k) { ++ auto s = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ p[k] = _mm256_madd_epi16(scales[k], _mm256_maddubs_epi16(s, _mm256_sign_epi8(q8.load_quants(iy, i, k), bits.values[k]))); ++ } ++ sumi[iy] = _mm256_add_epi32(_mm256_add_epi32(p[0], p[1]), _mm256_add_epi32(p[2], p[3])); ++ } ++ } else { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ for (int k = 0; k < 4; ++k) { ++ auto s = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ p[k] = _mm256_madd_epi16(scales[k], _mm256_maddubs_epi16(s, _mm256_sign_epi8(q8.load_quants(iy, i, 4+k), bits.values[k]))); ++ } ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p[0], p[2])); ++ sumi[iy] = _mm256_add_epi32(sumi[iy], _mm256_add_epi32(p[1], p[3])); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++ ++struct BlockPermuter { ++ const __m512i permute1 = _mm512_set_epi64(11, 10, 9, 8, 3, 2, 1, 0); ++ const __m512i permute2 = _mm512_set_epi64(15, 14, 13, 12, 7, 6, 5, 4); ++}; ++ ++struct Q4Bits { ++ inline void prepare(const uint8_t * q4) { ++ auto q4bits = _mm512_loadu_si512((const __m512i*)q4 + 0); ++ auto tmp1 = _mm512_and_si512(q4bits, ml); ++ auto tmp2 = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ values[0] = _mm512_permutex2var_epi64(tmp1, perm.permute1, tmp2); ++ values[1] = _mm512_permutex2var_epi64(tmp1, perm.permute2, tmp2); ++ q4bits = _mm512_loadu_si512((const __m512i*)q4 + 1); ++ tmp1 = _mm512_and_si512(q4bits, ml); ++ tmp2 = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ values[2] = _mm512_permutex2var_epi64(tmp1, perm.permute1, tmp2); ++ values[3] = _mm512_permutex2var_epi64(tmp1, perm.permute2, tmp2); ++ } ++ inline void prepare64(const uint8_t * q4) { ++ auto q4bits = _mm512_loadu_si512((const __m512i*)q4 + 0); ++ values[0] = _mm512_and_si512(q4bits, ml); ++ values[1] = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm512_loadu_si512((const __m512i*)q4 + 1); ++ values[2] = _mm512_and_si512(q4bits, ml); ++ values[3] = _mm512_and_si512(_mm512_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare64a(const uint8_t * q4) { ++ for (int k = 0; k < 4; ++k) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + k); ++ values[k] = _mm512_inserti32x8(_mm512_castsi256_si512(q4bits), _mm256_srli_epi16(q4bits, 4), 1); ++ values[k] = _mm512_and_si512(values[k], ml); ++ } ++ } ++ __m512i values[4]; ++ const __m512i ml = _mm512_set1_epi8(0xf); ++ const BlockPermuter perm; ++}; ++ ++struct Q2Bits { ++ inline void prepare(const uint8_t * q2) { ++ ++ auto q2bits = _mm512_loadu_si512((const __m512i*)q2); ++ auto tmp = _mm512_srli_epi16(q2bits, 2); ++ ++ values[0] = _mm512_permutex2var_epi64(q2bits, perm.permute1, tmp); ++ values[2] = _mm512_permutex2var_epi64(q2bits, perm.permute2, tmp); ++ values[1] = _mm512_and_si512(_mm512_srli_epi16(values[0], 4), ml); ++ values[3] = _mm512_and_si512(_mm512_srli_epi16(values[2], 4), ml); ++ values[0] = _mm512_and_si512(values[0], ml); ++ values[2] = _mm512_and_si512(values[2], ml); ++ } ++ __m512i values[4]; ++ const __m512i ml = _mm512_set1_epi8(0x03); ++ BlockPermuter perm; ++}; ++ ++#else ++ ++struct Q2Bits { ++ inline void prepare(const uint8_t * q2, int j) { ++ auto q2bits = _mm256_loadu_si256((const __m256i *)q2 + j); ++ values[0] = _mm256_and_si256(q2bits, ml); ++ values[1] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), ml); ++ values[2] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), ml); ++ } ++ __m256i values[4]; ++ const __m256i ml = _mm256_set1_epi8(0x03); ++}; ++ ++struct Q4Bits { ++ inline void prepare(const uint8_t * q4, int j) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+0); ++ values[0] = _mm256_and_si256(q4bits, ml); ++ values[1] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+1); ++ values[2] = _mm256_and_si256(q4bits, ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare64(const uint8_t * q4, int j) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+0); ++ values[0] = _mm256_and_si256(q4bits, ml); ++ values[2] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+1); ++ values[1] = _mm256_and_si256(q4bits, ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ } ++ inline void prepare16(const uint8_t * q4, int j) { ++ values[0] = dequant16(q4 + 64*j + 0); ++ values[1] = dequant16(q4 + 64*j + 16); ++ values[2] = dequant16(q4 + 64*j + 32); ++ values[3] = dequant16(q4 + 64*j + 48); ++ } ++ inline __m256i dequant16(const uint8_t * qs) const { ++ const __m128i aux128 = _mm_loadu_si128((const __m128i *)qs); ++ const __m256i aux256 = MM256_SET_M128I(_mm_srli_epi16(aux128, 4), aux128); ++ return _mm256_and_si256(ml, aux256); ++ } ++ __m256i values[4]; ++ const __m256i ml = _mm256_set1_epi8(0xf); ++}; ++ ++#endif ++ ++inline void iqk_transpose_8x8(__m256 * m) { ++ for (int k = 0; k < 8; k += 4) { ++ auto t0 = _mm256_unpacklo_ps(m[k+0], m[k+1]); ++ auto t1 = _mm256_unpacklo_ps(m[k+2], m[k+3]); ++ auto t2 = _mm256_unpackhi_ps(m[k+0], m[k+1]); ++ auto t3 = _mm256_unpackhi_ps(m[k+2], m[k+3]); ++ m[k+0] = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(t0), _mm256_castps_pd(t1))); ++ m[k+1] = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(t0), _mm256_castps_pd(t1))); ++ m[k+2] = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(t2), _mm256_castps_pd(t3))); ++ m[k+3] = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(t2), _mm256_castps_pd(t3))); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto t = _mm256_set_m128(_mm256_extractf128_ps(m[k+4], 1), _mm256_extractf128_ps(m[k], 1)); ++ m[k+0] = _mm256_set_m128(_mm256_castps256_ps128(m[k+4]), _mm256_castps256_ps128(m[k+0])); ++ m[k+4] = t; ++ } ++} ++ ++template ++static inline float convert_to_q8_k_r8(int k, float d0, const __m256i * qx, const int16_t * scales, uint32_t * block, int8_t * q8_k) { ++ auto max_i16 = _mm256_setzero_si256(); ++ __m256i qs[16]; ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ qs[2*ib32+0] = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(qx[ib32])); ++ qs[2*ib32+1] = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(qx[ib32], 1)); ++ qs[2*ib32+0] = _mm256_mullo_epi16(qs[2*ib32+0], _mm256_set1_epi16(scales[2*ib32+0])); ++ qs[2*ib32+1] = _mm256_mullo_epi16(qs[2*ib32+1], _mm256_set1_epi16(scales[2*ib32+1])); ++ max_i16 = _mm256_max_epi16(max_i16, _mm256_sign_epi16(qs[2*ib32+0], qs[2*ib32+0])); ++ max_i16 = _mm256_max_epi16(max_i16, _mm256_sign_epi16(qs[2*ib32+1], qs[2*ib32+1])); ++ } ++ auto max_q32 = _mm256_cvtepi16_epi32(_mm_max_epi16(_mm256_castsi256_si128(max_i16), _mm256_extracti128_si256(max_i16, 1))); ++ auto imax4 = _mm_max_epi32(_mm256_castsi256_si128(max_q32), _mm256_extracti128_si256(max_q32, 1)); ++ auto max4 = _mm_cvtepi32_ps(imax4); ++ max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); ++ max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4)); ++ bool needs_scaling = true; ++ float dnew = _mm_cvtss_f32(max4) * d0; ++ if (dnew < 1.f) { ++ dnew = 1.f; needs_scaling = false; ++ } ++ auto scale = _mm256_set1_ps(std::abs(dnew) > 1e-9f ? 1/dnew : 0.f); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ if (needs_scaling) { ++ auto i0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(qs[2*ib32+0])); ++ auto i1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(qs[2*ib32+0], 1)); ++ auto i2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(qs[2*ib32+1])); ++ auto i3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(qs[2*ib32+1], 1)); ++ i0 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i0)), _MM_ROUND_NEAREST)); ++ i1 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i1)), _MM_ROUND_NEAREST)); ++ i2 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i2)), _MM_ROUND_NEAREST)); ++ i3 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i3)), _MM_ROUND_NEAREST)); ++ i0 = _mm256_packs_epi32(i0, i1); ++ i2 = _mm256_packs_epi32(i2, i3); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)); ++ _mm256_storeu_si256((__m256i *)block, i0); ++ } else { ++ // 0, 1, 2, 3, 4, 5, 6, 7, 8, 16, 17, 18, 19, 20, 21, 22, 23, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 ++ auto i0 = _mm256_packs_epi16(qs[2*ib32+0], qs[2*ib32+1]); ++ auto i0_l = _mm256_castsi256_si128(i0); ++ auto i0_h = _mm256_extracti128_si256(i0, 1); ++ _mm_storeu_si128((__m128i *)block+0, _mm_unpacklo_epi64(i0_l, i0_h)); ++ _mm_storeu_si128((__m128i *)block+1, _mm_unpackhi_epi64(i0_l, i0_h)); ++ } ++ auto qs = (uint32_t *)q8_k + 8*nr*ib32; ++ for (int l = 0; l < 8; ++l) { ++ qs[nr*l + k] = block[l]; ++ } ++ } ++ return dnew; ++} ++ ++#else ++// ------------------------------------ __aarch64__ -------------------------------------------------- ++ ++template struct Q8 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q8(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8 *)info.src1_row(iy); ++ } ++ ++ inline int8x16x2_t load_quants(int iy, int i, int j) const { return vld1q_s8_x2(y[iy][i].qs + 32*j); } ++ inline int8x16x4_t load_quants_64(int iy, int i, int j) const { return vld1q_s8_x4(y[iy][i].qs + 64*j); } ++ inline int16x8x2_t load_bsums(int iy, int i) const { return vld1q_s16_x2(y[iy][i].bsums); } ++ inline int16x8_t load_bsums8(int iy, int i) const { ++ auto q8s = vld1q_s16_x2(y[iy][i].bsums); ++ return vpaddq_s16(q8s.val[0], q8s.val[1]); ++ } ++ inline float scale(int iy, int i) const { return y[iy][i].d; } ++ ++ const block_q8 * y[nrc_y]; ++}; ++ ++template ++struct BaseDequantizer { ++ BaseDequantizer(const void * vx, size_t bx, int nrc) : vx(vx), x(nullptr), bx(bx), nrc(nrc) {} ++ inline void new_row(int ix) { ++ if constexpr (has_row_scale) { ++ if constexpr (scale_is_f16) { ++ const ggml_half * dptr = (const ggml_half *)((const char *)vx + ix*bx); ++ d = GGML_FP16_TO_FP32(*dptr); ++ x = (const block_q *)(dptr + 1); ++ } else { ++ const float * dptr = (const float *)((const char *)vx + ix*bx); ++ d = *dptr; ++ x = (const block_q *)(dptr + 1); ++ } ++ } else { ++ x = (const block_q *)((const char *)vx + ix*bx); ++ } ++ } ++ const void * vx; ++ const block_q * x; ++ const size_t bx; ++ const int nrc; ++ float d; ++}; ++ ++struct Q4bits { ++ const uint8x16_t m4b = vdupq_n_u8(0xf); ++ uint8x16x4_t b1, b2; ++ inline void prepare4(uint8x16x4_t& b, const uint8x16_t * val) const { ++ b.val[0] = vandq_u8(val[0], m4b); ++ b.val[2] = vshrq_n_u8(val[0], 4); ++ b.val[1] = vandq_u8(val[1], m4b); ++ b.val[3] = vshrq_n_u8(val[1], 4); ++ } ++ inline void prepare4_16(uint8x16x4_t& b, const uint8x16_t * val) const { ++ b.val[0] = vandq_u8(val[0], m4b); ++ b.val[1] = vshrq_n_u8(val[0], 4); ++ b.val[2] = vandq_u8(val[1], m4b); ++ b.val[3] = vshrq_n_u8(val[1], 4); ++ } ++ inline void prepare(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x2(qs); ++ prepare4(b1, q4bits.val); ++ q4bits = vld1q_u8_x2(qs+32); ++ prepare4(b2, q4bits.val); ++ } ++ inline void prepare_v2(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ prepare4(b1, q4bits.val+0); ++ prepare4(b2, q4bits.val+2); ++ } ++ inline void prepare64(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ b1.val[0] = vandq_u8(q4bits.val[0], m4b); ++ b1.val[1] = vandq_u8(q4bits.val[1], m4b); ++ b1.val[2] = vandq_u8(q4bits.val[2], m4b); ++ b1.val[3] = vandq_u8(q4bits.val[3], m4b); ++ b2.val[0] = vshrq_n_u8(q4bits.val[0], 4); ++ b2.val[1] = vshrq_n_u8(q4bits.val[1], 4); ++ b2.val[2] = vshrq_n_u8(q4bits.val[2], 4); ++ b2.val[3] = vshrq_n_u8(q4bits.val[3], 4); ++ } ++ inline void prepare16(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x2(qs); ++ prepare4_16(b1, q4bits.val); ++ q4bits = vld1q_u8_x2(qs+32); ++ prepare4_16(b2, q4bits.val); ++ } ++ inline void prepare16_v2(const uint8_t * qs) { ++ auto q4bits = vld1q_u8_x4(qs); ++ prepare4_16(b1, q4bits.val+0); ++ prepare4_16(b2, q4bits.val+2); ++ } ++}; ++ ++struct Q2bits { ++ const uint8x16_t m4b = vdupq_n_u8(0x03); ++ uint8x16x4_t b1, b2; ++ inline void prepare(const uint8_t * qs) { ++ auto q2bits = vld1q_u8_x2(qs); ++ b1.val[0] = vandq_u8(q2bits.val[0], m4b); ++ b1.val[1] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b1.val[2] = vandq_u8(q2bits.val[0], m4b); ++ b1.val[3] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b2.val[0] = vandq_u8(q2bits.val[0], m4b); ++ b2.val[1] = vandq_u8(q2bits.val[1], m4b); ++ ++ q2bits.val[0] = vshrq_n_u8(q2bits.val[0], 2); ++ q2bits.val[1] = vshrq_n_u8(q2bits.val[1], 2); ++ b2.val[2] = vandq_u8(q2bits.val[0], m4b); ++ b2.val[3] = vandq_u8(q2bits.val[1], m4b); ++ } ++}; ++ ++template ++static inline void compute_8_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8, ++ const int32x4x2_t& scales, int iy, int i, int j, int32x4_t& sumi) { ++ auto mzero = vdupq_n_s32(0); ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ auto p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]), ++ vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1]); // block 1 ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ auto p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]), ++ vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1]); // block 2 ++ auto p12 = vpaddq_s32(p1, p2); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ auto p3 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]), ++ vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1]); // block 1 ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ auto p4 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]), ++ vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1]); // block 2 ++ auto p34 = vpaddq_s32(p3, p4); ++ ++ auto pall = vpaddq_s32(p12, p34); ++ sumi = vmlaq_s32(sumi, scales.val[j], pall); ++} ++ ++template ++static inline void compute_16_blocks(const uint8x16x4_t& qx_1, const uint8x16x4_t& qx_2, const Q8& q8, ++ const int32x4x4_t& scales, int iy, int i, int j, int32x4_t& sumi) { ++ ++ auto mzero = vdupq_n_s32(0); ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ auto p1 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[0]), q8b_1.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[1]), q8b_1.val[1])); // blocks 0, 0, 1, 1, ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ auto p2 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[2]), q8b_2.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_1.val[3]), q8b_2.val[1])); // blocks 3, 3, 4, 4, ++ auto p12 = vpaddq_s32(p1, p2); // blocks 0, 1, 2, 3 ++ sumi = vmlaq_s32(sumi, scales.val[2*j+0], p12); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ auto p3 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[0]), q8b_3.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[1]), q8b_3.val[1])); // block 4, 4, 5, 5, ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ auto p4 = vpaddq_s32(ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[2]), q8b_4.val[0]), ++ ggml_vdotq_s32(mzero, vreinterpretq_s8_u8(qx_2.val[3]), q8b_4.val[1])); // block 6, 6, 7, 7, ++ auto p34 = vpaddq_s32(p3, p4); // blocks 4, 5, 6, 7 ++ sumi = vmlaq_s32(sumi, scales.val[2*j+1], p34); ++} ++ ++struct SignHelper { ++ ++ inline void init() { shuffle = vcombine_u8(vdup_n_u8(0), vdup_n_u8(1)); } ++ ++ inline void apply_signs_1(uint8x16_t * b, const uint8x16_t& signs16) { ++ auto aux = vqtbl1q_u8(signs16, shuffle); ++ auto s = vreinterpretq_s8_u8(vorrq_u8(vceqq_u8(vandq_u8(aux, smask), smask), m1)); ++ b[0] = vreinterpretq_u8_s8(vmulq_s8(vreinterpretq_s8_u8(b[0]), s)); ++ shuffle = vaddq_u8(shuffle, step); ++ } ++ ++ const uint8x16_t smask = vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201)); ++ const uint8x16_t m1 = vdupq_n_u8(1); ++ const uint8x16_t step = vdupq_n_u8(2); ++ uint8x16_t shuffle; ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx, nrc_y); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ float32x4_t acc[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ int32x4_t sumi[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) sumi[iy] = vdupq_n_s32(0); ++ ++ if constexpr (nrc_y > 1 && Dequantizer::should_scale_quants()) { ++ deq.process_scales(i, q8, acc); ++ deq.prepare(i, 0); ++ deq.compute(q8, i, 0, sumi); ++ deq.prepare(i, 1); ++ deq.compute(q8, i, 1, sumi); ++ } else { ++ if constexpr (Dequantizer::num_blocks() == 8) { ++ auto scales = deq.new_block(i, q8, acc); ++ deq.prepare(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]); ++ deq.prepare(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_8_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]); ++ } ++ else if constexpr (Dequantizer::num_blocks() == 16) { ++ auto scales = deq.new_block(i, q8, acc); ++ deq.prepare(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 0, sumi[iy]); ++ deq.prepare(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) compute_16_blocks(deq.bits.b1, deq.bits.b2, q8, scales, iy, i, 1, sumi[iy]); ++ } ++ else { ++ GGML_ASSERT(false); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vmlaq_f32(acc[iy], vcvtq_f32_s32(sumi[iy]), vdupq_n_f32(deq.d*q8.scale(iy, i))); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, vaddvq_f32(acc[iy])); ++ } ++ } ++} ++ ++static IQK_ALWAYS_INLINE int32x4_t interleaved_dotq(const int8x16_t * qx, const int8x16x2_t& y) { ++ auto sumi = vdupq_n_s32(0); ++ sumi = vdotq_laneq_s32(sumi, qx[0], y.val[0], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[1], y.val[1], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[2], y.val[0], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[3], y.val[1], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[4], y.val[0], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[5], y.val[1], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[6], y.val[0], 3); ++ sumi = vdotq_laneq_s32(sumi, qx[7], y.val[1], 3); ++ return sumi; ++} ++ ++static IQK_ALWAYS_INLINE int32x4x2_t interleaved_dotq_b16(const int8x16_t * qx, const int8x16x2_t& y) { ++ int32x4x2_t sumi = { vdupq_n_s32(0), vdupq_n_s32(0) }; ++ sumi.val[0] = vdotq_laneq_s32(sumi.val[0], qx[0], y.val[0], 0); ++ sumi.val[1] = vdotq_laneq_s32(sumi.val[1], qx[1], y.val[1], 0); ++ sumi.val[0] = vdotq_laneq_s32(sumi.val[0], qx[2], y.val[0], 1); ++ sumi.val[1] = vdotq_laneq_s32(sumi.val[1], qx[3], y.val[1], 1); ++ sumi.val[0] = vdotq_laneq_s32(sumi.val[0], qx[4], y.val[0], 2); ++ sumi.val[1] = vdotq_laneq_s32(sumi.val[1], qx[5], y.val[1], 2); ++ sumi.val[0] = vdotq_laneq_s32(sumi.val[0], qx[6], y.val[0], 3); ++ sumi.val[1] = vdotq_laneq_s32(sumi.val[1], qx[7], y.val[1], 3); ++ return sumi; ++} ++ ++static IQK_ALWAYS_INLINE int32x4_t interleaved_dotq(const int8x16_t * qx, const int8x16_t& y) { ++ auto sumi = vdupq_n_s32(0); ++ sumi = vdotq_laneq_s32(sumi, qx[0], y, 0); ++ sumi = vdotq_laneq_s32(sumi, qx[1], y, 1); ++ sumi = vdotq_laneq_s32(sumi, qx[2], y, 2); ++ sumi = vdotq_laneq_s32(sumi, qx[3], y, 3); ++ return sumi; ++} ++ ++static IQK_ALWAYS_INLINE void prepare_iq4_nl_quants(const int8x16_t& values, const uint8x16_t& m4, const uint8x16x4_t& bits, int8x16_t * qx) { ++ qx[0] = vqtbl1q_s8(values, vandq_u8(bits.val[0], m4)); // 0...3 from the 4 rows ++ qx[1] = vqtbl1q_s8(values, vandq_u8(bits.val[1], m4)); // 16..19 ++ qx[2] = vqtbl1q_s8(values, vandq_u8(bits.val[2], m4)); // 4...7 ++ qx[3] = vqtbl1q_s8(values, vandq_u8(bits.val[3], m4)); // 20..23 ++ qx[4] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[0], 4)); // 8..11 ++ qx[5] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[1], 4)); // 24..27 ++ qx[6] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[2], 4)); // 12..15 ++ qx[7] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[3], 4)); // 28..31 ++} ++ ++static IQK_ALWAYS_INLINE void prepare_iq4_nl_quants_r8(const int8x16_t& values, const uint8x16_t& m4, const uint8x16x2_t& bits, int8x16_t * qx) { ++ qx[0] = vqtbl1q_s8(values, vandq_u8( bits.val[0], m4)); ++ qx[1] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[0], 4)); ++ qx[2] = vqtbl1q_s8(values, vandq_u8( bits.val[1], m4)); ++ qx[3] = vqtbl1q_s8(values, vshrq_n_u8(bits.val[1], 4)); ++} ++ ++#endif ++ ++#endif ++ ++// static unrool for: ++template ++inline void static_for(T&&f) { ++ if constexpr(N>0) { ++ static_for(f); ++ f(N-1); ++ } ++} ++ ++#if defined(_MSC_VER) ++#pragma warning(disable: 4244 4267) // possible loss of data ++#include ++#include ++#include ++#include ++#include ++inline int popcount(uint8_t x) { return __popcnt(x); } ++inline int popcount(uint16_t x) { return __popcnt(x); } ++inline int popcount(uint32_t x) { return __popcnt(x); } ++inline int popcount(uint64_t x) { return _mm_popcnt_u64(x); } ++#else ++constexpr int popcount(uint8_t x) { return __builtin_popcount(x); } ++constexpr int popcount(uint16_t x) { return __builtin_popcount(x); } ++constexpr int popcount(uint32_t x) { return __builtin_popcount(x); } ++constexpr int popcount(uint64_t x) { return __builtin_popcountll(x); } ++#endif ++ +diff --git a/llama.cpp/ggml/src/iqk/iqk_common_extra.h b/llama.cpp/ggml/src/iqk/iqk_common_extra.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_common_extra.h +@@ -0,0 +1,471 @@ ++// opencoti F5-opt W2 (#290) — ik_llama.cpp ggml-common.h DELTA. ++// VENDORED from ik_llama.cpp @ 8960c5ba… ggml/src/ggml-common.h ++// Contains ONLY the block types ik_llama defines that llamafile's ggml-common.h ++// lacks (extracted compiler-driven: each entry was an "undeclared type" error). ++// MIT, (C) 2024 Iwan Kawrakow. Re-sync: diff upstream ggml-common.h block defs. ++#pragma once ++#ifndef IQK_COMMON_EXTRA_H ++#define IQK_COMMON_EXTRA_H ++ ++typedef struct { ++ ggml_half d[4]; // delta ++ uint8_t qh[QK5_0/2]; // 5-th bit of quants ++ uint8_t qs[QK5_0*2]; // nibbles / quants ++} block_q5_0_r4; ++static_assert(sizeof(block_q5_0_r4) == 4*sizeof(ggml_half) + QK5_0*2 + QK5_0/2, "wrong q5_0_r4 block size/padding"); ++ ++#define QK6_0 32 ++typedef struct { ++ ggml_half d; // delta ++ uint8_t qh[QK6_0/4]; // 5+6-th bit of quants ++ uint8_t qs[QK6_0/2]; // nibbles / quants ++} block_q6_0; ++static_assert(sizeof(block_q6_0) == sizeof(ggml_half) + QK6_0/2 + QK6_0/4, "wrong q6_0 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; // delta ++ uint8_t qh[QK6_0]; // 5+6-th bit of quants ++ uint8_t qs[QK6_0*2]; // nibbles / quants ++} block_q6_0_r4; ++static_assert(sizeof(block_q6_0_r4) == 4*sizeof(ggml_half) + QK6_0*2 + QK6_0, "wrong q6_0_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ int8_t qs[4*QK8_1]; ++} block_q8_1_x4; ++static_assert(sizeof(block_q8_1_x4) == 4*sizeof(block_q8_1), "wrong q8_1_x4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ int8_t qs[4*QK8_0]; ++} block_q8_0_x4; ++static_assert(sizeof(block_q8_0_x4) == 4*sizeof(block_q8_0), "wrong q8_0_x4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ int8_t qs[8*QK8_0]; ++} block_q8_0_r8; ++static_assert(sizeof(block_q8_0_r8) == 8*sizeof(block_q8_0), "wrong q8_0_r8 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; // deltas for 4 q4_0 blocks ++ uint8_t qs[QK4_0 * 2]; // nibbles / quants for 4 q4_0 blocks ++} block_q4_0x4; ++static_assert(sizeof(block_q4_0x4) == 4 * sizeof(ggml_half) + QK4_0 * 2, "wrong q4_0x4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; // deltas for 8 q4_0 blocks ++ uint8_t qs[QK4_0 * 4]; // nibbles / quants for 8 q4_0 blocks ++} block_q4_0x8; ++static_assert(sizeof(block_q4_0x8) == 8 * sizeof(ggml_half) + QK4_0 * 4, "wrong q4_0x8 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; // deltas for 4 q8_0 blocks ++ int8_t qs[QK8_0 * 4]; // quants for 4 q8_0 blocks ++} block_q8_0x4; ++static_assert(sizeof(block_q8_0x4) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong q8_0x4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; // deltas for 8 q8_0 blocks ++ int8_t qs[QK8_0 * 8]; // quants for 8 q8_0 blocks ++} block_q8_0x8; ++static_assert(sizeof(block_q8_0x8) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding"); ++ ++#define QK8_2 32 ++typedef struct { ++ uint16_t d; ++ uint16_t s; ++ int8_t qs[QK8_2]; // quants ++} block_q8_2; ++static_assert(sizeof(block_q8_2) == sizeof(ggml_half) + sizeof(int16_t) + QK8_2, "wrong q8_2 block size/padding"); ++ ++typedef struct { ++ uint16_t d[8]; ++ int8_t qs[4*QK8_2]; ++} block_q8_2_x4; ++static_assert(sizeof(block_q8_2_x4) == 4*sizeof(block_q8_2), "wrong q8_2_x4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ uint8_t scales[QK_K/4]; // scales and mins, quantized with 4 bits ++ uint8_t qs[QK_K]; // quants ++} block_q2_k_r4; ++static_assert(sizeof(block_q2_k_r4) == 8*sizeof(ggml_half) + QK_K/4 + QK_K, "wrong q2_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; // super-block scales ++ uint8_t scales_h[QK_K/16]; // scales quantized with 6 bits (high 2 bits) ++ uint8_t scales_l[QK_K/8]; // scales quantized with 6 bits (low 4 bits) ++ uint8_t qh[QK_K/2]; // quants - high bit ++ uint8_t qs[QK_K]; // quants - low 2 bits ++} block_q3_k_r4; ++static_assert(sizeof(block_q3_k_r4) == 4*sizeof(ggml_half) + QK_K/16 + QK_K/8 + QK_K/2 + QK_K, "wrong q3_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ uint8_t scales_h[QK_K/16];// scales and mins, quantized with 6 bits ++ uint8_t scales_l[QK_K/8]; // scales and mins, quantized with 6 bits ++ uint8_t qs[QK_K*2]; // 4--bit quants ++} block_q4_k_r4; ++static_assert(sizeof(block_q4_k_r4) == 8*sizeof(ggml_half) + QK_K/16 + QK_K/8 + QK_K*2, "wrong q4_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ uint8_t scales_h[QK_K/16];// scales and mins, quantized with 6 bits ++ uint8_t scales_l[QK_K/8]; // scales and mins, quantized with 6 bits ++ uint8_t qh[QK_K/2]; // quants, high bit ++ uint8_t qs[QK_K*2]; // quants, low 4 bits ++} block_q5_k_r4; ++static_assert(sizeof(block_q5_k_r4) == 8*sizeof(ggml_half) + QK_K/16 + QK_K/8 + QK_K/2 + QK_K*2, "wrong q5_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; // super-block scale ++ int8_t scales[QK_K/4]; // scales, quantized with 8 bits ++ uint8_t qh[QK_K]; // quants, upper 2 bits ++ uint8_t ql[QK_K*2]; // quants, lower 4 bits ++} block_q6_k_r4; ++static_assert(sizeof(block_q6_k_r4) == 4*sizeof(ggml_half) + QK_K/4 + 3*QK_K, "wrong q6_k_r4 block size/padding"); ++ ++typedef struct { ++ float d; // delta ++ int8_t qs[64]; // quants ++} block_q8_K64; ++static_assert(sizeof(block_q8_K64) == sizeof(float) + 64, "wrong q8_K64 block size/padding"); ++ ++typedef struct { ++ float d; // delta ++ int16_t bsums[4]; // quant sums for blocks of 32 ++ int8_t qs[128]; // quants ++} block_q8_K128; ++static_assert(sizeof(block_q8_K128) == sizeof(float) + 4*sizeof(int16_t) + 128, "wrong q8_K128 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; // delta ++ int8_t qs[8*QK_K]; // quants, stored as unsigned ints ++} block_q8_k_r8; ++static_assert(sizeof(block_q8_k_r8) == 8*sizeof(ggml_half) + 8*QK_K, "wrong q8_k_r8 block size/padding"); ++ ++typedef struct { ++ ggml_half d[16]; // delta ++ int8_t qs[16*QK_K]; // quants, stored as unsigned ints ++} block_q8_k_r16; ++static_assert(sizeof(block_q8_k_r16) == 16*sizeof(ggml_half) + 16*QK_K, "wrong q8_k_r16 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t sas[QK_K/2]; ++ uint8_t qs[QK_K/2]; ++} block_iq2_xxs_r4; ++static_assert(sizeof(block_iq2_xxs_r4) == 4*sizeof(block_iq2_xxs), "wrong iq2_xxs_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint16_t qs[QK_K/2]; ++ uint8_t scales[QK_K/8]; ++} block_iq2_xs_r4; ++static_assert(sizeof(block_iq2_xs_r4) == 4*sizeof(block_iq2_xs), "wrong iq2_xs_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t qs[QK_K/2]; ++ uint8_t qh[QK_K/8]; ++ uint8_t signs[QK_K/2]; ++ uint8_t scales[QK_K/8]; ++} block_iq2_s_r4; ++static_assert(sizeof(block_iq2_s_r4) == 4*sizeof(block_iq2_s), "wrong iq2_s_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t sas[QK_K/2]; ++ uint8_t qs[QK_K]; ++} block_iq3_xxs_r4; ++static_assert(sizeof(block_iq3_xxs_r4) == 4*sizeof(block_iq3_xxs), "wrong iq3_xxs_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t qs[QK_K]; ++ uint8_t qh[QK_K/8]; ++ uint8_t signs[QK_K/2]; ++ uint8_t scales[4*IQ3S_N_SCALE]; ++} block_iq3_s_r4; ++static_assert(sizeof(block_iq3_s_r4) == 4*sizeof(block_iq3_s), "wrong iq3_s_r4 block size/padding"); ++ ++typedef struct { ++ uint8_t qs[16]; ++ uint16_t qh[4]; ++} block_iq1_s_r4; ++static_assert(sizeof(block_iq1_s_r4) == 24, "wrong iq1_s_r4 block size/padding"); ++ ++// 1.75 bpw - blocks of 32 with 4 interleaved rows = 128 quants ++typedef struct { ++ uint8_t qs[16]; // grid index, low 8 bits ++ uint8_t qh[ 8]; // grid index, high 3 bits + grid shift bits (for two groups of 8) ++ uint8_t scales[4]; // 4-bit block scales ++} block_iq1_m_r4; ++static_assert(sizeof(block_iq1_m_r4) == 28, "wrong iq1_m_r4 block size/padding"); ++ ++// ++// Bonsai ++// ++#define QK1_0_G128 128 ++typedef struct { ++ ggml_half d; ++ uint8_t qs[QK1_0_G128 / 8]; ++} block_q1_0_g128; ++static_assert(sizeof(block_q1_0_g128) == sizeof(ggml_half) + QK1_0_G128 / 8, "wrong q1_0_g128 block size/padding"); ++ ++// ++// Bitnet and TriLM - implemented as 1.625 bpw ++// ++#define QK_IQ1BN 64 ++typedef struct { ++ uint8_t ql[12]; ++ uint8_t extra; ++} block_iq1_bn; ++static_assert(sizeof(block_iq1_bn) == 13, "wrong iq1_bn block size/padding"); ++ ++// ++// Bitnet and TriLM - implemented as 2.0 bpw ++// ++#define QK_IQ2BN 64 ++typedef struct { ++ uint8_t qs[QK_IQ2BN/4]; ++} block_iq2_bn; ++static_assert(sizeof(block_iq2_bn) == QK_IQ2BN/4, "wrong iq2_bn block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t qs[2*QK4_NL]; ++} block_iq4_nl_r4; ++static_assert(sizeof(block_iq4_nl_r4) == 4*sizeof(ggml_half) + 2*QK4_NL, "wrong iq4_nl_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ uint8_t qs[4*QK4_NL]; ++} block_iq4_nl_r8; ++static_assert(sizeof(block_iq4_nl_r8) == 8*sizeof(ggml_half) + 4*QK4_NL, "wrong iq4_nl_r8 block size/padding"); ++ ++typedef struct { ++ ggml_half d[8]; ++ uint8_t scales_h[QK_K/16]; ++ uint8_t scales_l[QK_K/ 8]; ++ uint8_t qs[QK_K*4]; ++} block_iq4_xs_r8; ++static_assert(sizeof(block_iq4_xs_r8) == 8*sizeof(block_iq4_xs), "wrong iq4_xs_rs block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/32]; ++ uint8_t qs[QK_K/2]; ++} block_iq4_ks; ++static_assert(sizeof(block_iq4_ks) == QK_K/32 + QK_K/2, "wrong iq4_ks block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/8]; ++ uint8_t qs[QK_K*2]; ++} block_iq4_ks_r4; ++static_assert(sizeof(block_iq4_ks_r4) == 4*sizeof(block_iq4_ks), "wrong iq4_ks_r4 block size/padding"); ++ ++typedef struct { ++ uint32_t qs[QK_K/8]; ++} block_iq4_kss; ++static_assert(sizeof(block_iq4_kss) == QK_K/8*sizeof(uint32_t), "wrong iq4_kss block size/padding"); ++ ++typedef struct { ++ ggml_half d; ++ uint16_t extra; ++ uint8_t scales[QK_K/32]; ++ uint8_t qs[QK_K/4]; ++} block_iq2_k; ++static_assert(sizeof(block_iq2_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/32 + QK_K/4, "wrong iq2_k block size/padding"); ++ ++typedef struct { ++ uint16_t scales_h; ++ uint8_t scales_l[QK_K/64]; ++ uint8_t qs[QK_K/4]; ++ uint8_t qh[QK_K/16]; ++} block_iq2_kl; ++static_assert(sizeof(block_iq2_kl) == sizeof(uint16_t) + QK_K/64 + QK_K/4 + QK_K/16, "wrong iq2_kl block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t extra[8]; ++ uint8_t scales[QK_K/8]; ++ uint8_t qs[QK_K]; ++} block_iq2_k_r4; ++static_assert(sizeof(block_iq2_k_r4) == 4*sizeof(block_iq2_k), "wrong iq2_k_r4 block size/padding"); ++ ++typedef struct { ++ uint16_t extra; ++ uint8_t scales[QK_K/64]; ++ uint8_t qs[QK_K/4]; ++} block_iq2_ks; ++static_assert(sizeof(block_iq2_ks) == sizeof(uint16_t) + QK_K/64 + QK_K/4, "wrong iq2_ks block size/padding"); ++ ++typedef struct { ++ uint8_t sh[QK_K/32]; // 4-bit scales + 13th bits for groups of 8 ++ uint8_t ql[QK_K/8]; // low 8 bits for groups of 8 ++ uint8_t qh[QK_K/16]; // high 4 bits for groups of 8 ++} block_iq1_kt; ++static_assert(sizeof(block_iq1_kt) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_kt block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/64]; ++ uint8_t ql[QK_K/4]; ++} block_iq2_kt; ++static_assert(sizeof(block_iq2_kt) == QK_K/4 + QK_K/64, "wrong iq2_kt block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/64]; ++ uint8_t ql[QK_K/4]; ++ uint8_t qh[QK_K/8]; ++} block_iq3_kt; ++static_assert(sizeof(block_iq3_kt) == QK_K/4 + QK_K/8 + QK_K/64, "wrong iq3_kt block size/padding"); ++ ++typedef struct { ++ uint32_t qs[QK_K/8]; ++} block_iq4_kt; ++static_assert(sizeof(block_iq4_kt) == QK_K/2, "wrong iq4_kt block size/padding"); ++ ++typedef struct { ++ ggml_half d; ++ uint16_t extra; ++ uint16_t scales_h; ++ uint8_t scales_l[QK_K/32]; ++ uint8_t qs[QK_K/4]; ++ uint8_t qh[QK_K/8]; ++} block_iq3_k; ++static_assert(sizeof(block_iq3_k) == sizeof(ggml_half) + 2*sizeof(uint16_t) + QK_K/32 + QK_K/4 + QK_K/8, "wrong iq3_k block size/padding"); ++ ++typedef struct { ++ uint16_t extra; ++ uint8_t scales[QK_K/64]; ++ uint8_t qs[QK_K/4]; ++ uint8_t qh[QK_K/8]; ++} block_iq3_ks; ++static_assert(sizeof(block_iq3_ks) == sizeof(uint16_t) + QK_K/64 + QK_K/4 + QK_K/8, "wrong iq3_ks block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t extra[8]; ++ uint8_t scales_h[QK_K/32]; ++ uint8_t scales_l[QK_K/8]; ++ uint8_t qs[QK_K]; ++ uint8_t qh[QK_K/2]; ++} block_iq3_k_r4; ++static_assert(sizeof(block_iq3_k_r4) == 4*sizeof(block_iq3_k), "wrong iq3_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d; ++ uint16_t extra; ++ uint8_t scales_h[QK_K/64]; ++ uint8_t scales_l[QK_K/32]; ++ uint8_t qs[QK_K/2]; ++} block_iq4_k; ++static_assert(sizeof(block_iq4_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/2 + 3*QK_K/64, "wrong iq4_k block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t extra[8]; ++ uint8_t scales_h[QK_K/16]; ++ uint8_t scales_l[QK_K/8]; ++ uint8_t qs[QK_K*2]; ++} block_iq4_k_r4; ++static_assert(sizeof(block_iq4_k_r4) == 4*sizeof(block_iq4_k), "wrong iq4_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d; ++ uint16_t extra; ++ uint8_t scales_h[QK_K/64]; ++ uint8_t scales_l[QK_K/32]; ++ uint8_t qs[QK_K/2]; ++ uint8_t qh[QK_K/8]; ++} block_iq5_k; ++static_assert(sizeof(block_iq5_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/2 + QK_K/8 + 3*QK_K/64, "wrong iq5_k block size/padding"); ++ ++typedef struct { ++ ggml_half d[4]; ++ uint8_t extra[8]; ++ uint8_t scales_h[QK_K/16]; ++ uint8_t scales_l[QK_K/8 ]; ++ uint8_t qs[QK_K*2]; ++ uint8_t qh[QK_K/2]; ++} block_iq5_k_r4; ++static_assert(sizeof(block_iq5_k_r4) == 4*sizeof(block_iq5_k), "wrong iq5_k_r4 block size/padding"); ++ ++typedef struct { ++ ggml_half d; ++ uint16_t extra; ++ int8_t scales[QK_K/16]; ++ uint8_t qs[QK_K/2]; ++ uint8_t qh[QK_K/4]; ++} block_iq6_k; ++static_assert(sizeof(block_iq6_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/2 + QK_K/4 + QK_K/16, "wrong iq6_k block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/32]; ++ uint8_t qs[QK_K/2]; ++ uint8_t qh[QK_K/8]; ++} block_iq5_ks; ++static_assert(sizeof(block_iq5_ks) == QK_K/32 + QK_K/2 + QK_K/8, "wrong iq5_ks block size/padding"); ++ ++typedef struct { ++ uint8_t scales[QK_K/8]; ++ uint8_t qs[QK_K*2]; ++ uint8_t qh[QK_K/2]; ++} block_iq5_ks_r4; ++static_assert(sizeof(block_iq5_ks_r4) == 4*sizeof(block_iq5_ks), "wrong iq5_ks_r4 block size/padding"); ++ ++ ++// opencoti F5-opt W2 (#290): ik_llama's block_q8_K carries an extra `float sum;` ++// after `d` (used only for intermediate quantization / dot products, never GGUF- ++// serialized). llamafile's GGUF-frozen block_q8_K has no such member and must not ++// be edited. iqk_quantize_row_q8_K_T() writes y[i].sum, so we give it an IK-private ++// struct with the ik_llama layout to cast to. Body verbatim from ik_llama ++// ggml-common.h; static_assert pins the (larger) ik layout. ++typedef struct { ++ float d; // delta ++ float sum; // sum of quants in the entire block ++ int8_t qs[QK_K]; // quants ++ int16_t bsums[QK_K/16]; // sum of quants in groups of 16 ++} iqk_block_q8_K; ++static_assert(sizeof(iqk_block_q8_K) == 2*sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong iqk_block_q8_K size/padding"); ++ ++// IK-only lookup table (ggml-common.h IMPL delta); static = per-TU copy, harmless. ++static const int8_t iq4k_values[32] = { ++-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, ++ -123, -100, -79, -61, -45, -31, -18, -6, 5, 17, 29, 42, 57, 73, 93, 117 ++}; ++ ++// opencoti F5-opt W2 (#290): more IK-only lookup tables (ggml-common.h IMPL deltas, ++// needed by iqk_quantize.cpp). static = per-TU copy, harmless. ++static const int8_t iq2nl_values[8] = { ++ -31, -13, 1, 17, -26, -8, 6, 22 ++}; ++ ++static const uint16_t iq2kl_values[32] = { ++ 0xe9c1, 0x0dc1, 0xc1d8, 0xf6d8, 0x0dd8, 0x2fd8, 0xd8e9, 0xe9e9, 0x01e9, 0x0de9, 0x1ce9, 0xc1f6, 0x01f6, 0x0df6, 0x2ff6, 0xe901, ++ 0xf601, 0x0101, 0x0d01, 0x1c01, 0xd80d, 0xe90d, 0xf60d, 0x010d, 0x0d0d, 0xc11c, 0xe91c, 0x011c, 0x1c1c, 0x2f1c, 0xe92f, 0x0d2f, ++}; ++ ++static const int8_t iq3nl_values[16] = { ++ -63, -40, -23, -10, 1, 13, 28, 47, ++ -59, -36, -19, -6, 5, 17, 32, 51, ++}; ++ ++static const int8_t iq5nl_values[64] = { ++ -126, -114, -103, -92, -83, -74, -65, -57, -50, -43, -36, -30, -24, -18, -12, -6, -1, 5, 11, 17, 23, 29, 36, 43, 51, 59, 68, 77, 87, 97, 109, 121, ++ -124, -112, -101, -90, -81, -72, -63, -55, -48, -41, -34, -28, -22, -16, -10, -4, 1, 7, 13, 19, 25, 31, 38, 45, 53, 61, 70, 79, 89, 99, 111, 123, ++}; ++ ++static const int8_t iq6nl_values[128] = { ++ -127, -121, -115, -109, -104, -98, -93, -88, -84, -79, -74, -70, -66, -62, -58, -54, ++ -51, -47, -44, -40, -37, -34, -31, -28, -25, -22, -19, -16, -13, -11, -8, -5, ++ -2, 0, 3, 6, 9, 12, 14, 17, 20, 23, 27, 30, 33, 36, 40, 44, ++ 47, 51, 55, 59, 63, 68, 72, 77, 82, 87, 92, 98, 103, 109, 115, 121, ++ -126, -120, -114, -108, -103, -97, -92, -87, -83, -78, -73, -69, -65, -61, -57, -53, ++ -50, -46, -43, -39, -36, -33, -30, -27, -24, -21, -18, -15, -12, -10, -7, -4, ++ -1, 1, 4, 7, 10, 13, 15, 18, 21, 24, 28, 31, 34, 37, 41, 45, ++ 48, 52, 56, 60, 64, 69, 73, 78, 83, 88, 93, 99, 104, 110, 116, 122, ++}; ++ ++#endif // IQK_COMMON_EXTRA_H +diff --git a/llama.cpp/ggml/src/iqk/iqk_config.h b/llama.cpp/ggml/src/iqk/iqk_config.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_config.h +@@ -0,0 +1,101 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++ ++#if defined IQK_IMPLEMENT ++#undef IQK_IMPLEMENT ++#endif ++ ++#if defined __AVX2__ || defined __ARM_FEATURE_DOTPROD ++#define IQK_IMPLEMENT ++#endif ++ ++// opencoti F5-opt W2 (#290): ik_llama's iqk headers (iqk_common.h / iqk_utils.h) ++// use __m256/__m128i in early inline code that, upstream, relies on ik_llama's ++// ggml-impl.h having pulled the SIMD intrinsics header first. llamafile's ++// ggml-impl.h does NOT (it surfaces intrinsics via ggml-cpu/ggml-cpu-impl.h, ++// which the iqk TUs don't include), so we pull them here — the root iqk config ++// header every iqk TU includes first. Guarded to the active ISA; non-x86/non-ARM ++// or non-AVX2 builds get nothing (IQK_IMPLEMENT is also off there). ++#if defined IQK_IMPLEMENT ++#if defined __x86_64__ || defined __i386__ ++#include ++#elif defined __aarch64__ || defined __ARM_NEON ++#include ++#endif ++#endif ++ ++// opencoti F5-opt W2 (#290): ggml_vdotq_s32 — ik_llama defines it in ggml-impl.h ++// (which the iqk TUs include), but llamafile MOVED it to ggml-cpu/ggml-cpu-impl.h ++// (which the iqk TUs do NOT include). iqk_quantize.cpp's #ifdef __ARM_NEON path ++// calls it, so under cosmocc's aarch64 slice (NEON on, DOTPROD off) the symbol is ++// missing. Re-supply the upstream-identical NEON fallback here, mirroring the ++// intrinsics shim above. Guarded by __ARM_NEON + sentinel so we never collide with ++// llamafile's ggml-cpu-impl.h should a TU ever pull it. ++#if defined __ARM_NEON && !defined GGML_VDOTQ_S32_DEFINED ++#define GGML_VDOTQ_S32_DEFINED ++#include ++#if !defined(__ARM_FEATURE_DOTPROD) ++inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) { ++ const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b)); ++ const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b)); ++ return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1))); ++} ++#else ++#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c) ++#endif // !defined(__ARM_FEATURE_DOTPROD) ++#endif // __ARM_NEON && !GGML_VDOTQ_S32_DEFINED ++ ++#ifdef GGML_SHARED ++# if defined(_WIN32) && !defined(__MINGW32__) ++# ifdef GGML_BUILD ++# define IQK_API __declspec(dllexport) ++# else ++# define IQK_API __declspec(dllimport) ++# endif ++# else ++# define IQK_API __attribute__ ((visibility ("default"))) ++# endif ++#else ++# define IQK_API ++#endif ++ ++#ifdef _MSC_VER ++#define IQK_NOINLINE __declspec(noinline) ++#define IQK_ALWAYS_INLINE inline ++#if !defined __x86_64__ && defined _M_X64 ++#define __x86_64__ ++#endif ++#else ++#define IQK_NOINLINE __attribute__((__noinline__)) ++#define IQK_ALWAYS_INLINE __attribute__((__always_inline__)) ++#endif ++ ++#if defined __x86_64__ ++#if defined HAVE_FANCY_SIMD ++ #undef HAVE_FANCY_SIMD ++#endif ++#if defined(__AVX512F__) && defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__) ++ #define HAVE_FANCY_SIMD ++#endif ++#if defined HAVE_VNNI256 ++ #undef HAVE_VNNI256 ++#endif ++#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__)) ++ #define HAVE_VNNI256 ++#endif ++#if defined(__AVX512VNNI__) && defined(__AVX512VL__) ++ #define ggml_mm256_dpbusd_epi32 _mm256_dpbusd_epi32 ++ #define ggml_mm256_dpwssd_epi32 _mm256_dpwssd_epi32 ++ #define ggml_mm_dpbusd_epi32 _mm_dpbusd_epi32 ++#elif defined(__AVXVNNI__) ++ #define ggml_mm256_dpbusd_epi32 _mm256_dpbusd_avx_epi32 ++ #define ggml_mm256_dpwssd_epi32 _mm256_dpwssd_avx_epi32 ++ #define ggml_mm_dpbusd_epi32 _mm_dpbusd_avx_epi32 ++#endif ++#endif ++ +diff --git a/llama.cpp/ggml/src/iqk/iqk_fa_dispatch.cpp b/llama.cpp/ggml/src/iqk/iqk_fa_dispatch.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_fa_dispatch.cpp +@@ -0,0 +1,230 @@ ++// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*- ++// vi: set et ft=cpp fenc=utf-8 :vi ++// ++// opencoti F5-opt W2 (#290) — CPU Flash-Attention dispatch + K-repack. ++// ++// VENDORED (extracted) from ik_llama.cpp ++// repo: https://github.com/ikawrakow/ik_llama.cpp ++// commit: 8960c5ba5ee9db30ba838304373aa4dbec9f7cbd ++// file: ggml/src/iqk/iqk_mul_mat.cpp lines 1249-1433 ++// (the `#ifdef GGML_IQK_FLASH_ATTENTION` block: iqk_repack_k + ++// iqk_flash_attn_impl, plus its inner template includes) ++// license: MIT — Copyright (C) 2024 Iwan Kawrakow ++// ++// WHY THIS FILE EXISTS (do not "fix" by merging back): ++// The two functions below are the ONLY parts of the 1780-LOC iqk_mul_mat.cpp ++// the FA engine needs. The rest of that TU is the universal GEMM dispatcher, ++// which (a) #includes + references the iquants/1bit/iqk_quants/ktquants GEMM ++// families (~800 KB of extra .cpp we do NOT want), and (b) defines the ++// extern-C symbols `iqk_mul_mat` / `iqk_mul_mat_moe` that COLLIDE with ++// llamafile's own tinyBLAS `iqk_mul_mat.inc`. Extracting just these two ++// functions lets us drop iqk_mul_mat.cpp entirely — no GEMM-family balloon, ++// no ODR collision, no symbol rename needed. ++// ++// To re-sync: re-extract lines 1249-1433 of upstream iqk_mul_mat.cpp at the ++// pinned SHA above (boundaries = the GGML_IQK_FLASH_ATTENTION guard pair). ++// ++#include "iqk_config.h" ++ ++#if defined IQK_IMPLEMENT ++ ++#include ++#include ++#include ++#include ++ ++#include "ggml-impl.h" ++#include "ggml-quants.h" ++#include "ggml-common.h" ++#include "iqk_mul_mat.h" ++#include "iqk_quantize.h" ++#include "iqk_utils.h" ++#include "iqk_common.h" ++ ++#ifdef GGML_IQK_FLASH_ATTENTION ++ ++void * iqk_repack_k(int int_type_k, int nek0, int nek1, int nek2, int nek3, long nbk1, long nbk2, long nbk3, ++ const void * data, void * work, int ith, int nth, int& repacked_type, uint64_t& row_size) { ++ repacked_type = int_type_k; ++ auto type_k = ggml_type(int_type_k); ++ if (type_k != GGML_TYPE_Q8_0 || nek0%QK8_0 != 0) return work; ++ int nrows = nek1*nek2*nek3; ++ if (nrows%8 != 0) return work; ++ repacked_type = int(GGML_TYPE_Q8_0_R8); ++ row_size = ggml_row_size(GGML_TYPE_Q8_0, nek0); ++ void * result = (char *)work + nrows*row_size; ++ int npt = 8*((nrows/8 + nth - 1)/nth); ++ int first = npt*ith; ++ if (first >= nrows) return result; ++ int last = std::min(first + npt, nrows); ++ const block_q8_0 * x8[8]; ++ auto y = (block_q8_0_r8 *)((char *)work + first*row_size); ++ int nblock = nek0/QK8_0; ++#ifdef __ARM_NEON ++ int8x16x2_t m0, m1, m2, m3; ++#endif ++ for (int row = first; row < last; row += 8) { ++ int ik3 = row/(nek1*nek2); ++ int ik2 = (row - ik3*nek1*nek2)/nek1; ++ int ik1 = row - ik3*nek1*nek2 - ik2*nek1; ++ auto this_data = (const char *)data + ik1*nbk1 + ik2*nbk2 + ik3*nbk3; ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q8_0 *)(this_data + k*nbk1); ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 8; ++k) y[ib].d[k] = x8[k][ib].d; ++#ifdef __AVX2__ ++ auto m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4][ib].qs), _mm_loadu_si128((const __m128i *)x8[0][ib].qs)); ++ auto m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5][ib].qs), _mm_loadu_si128((const __m128i *)x8[1][ib].qs)); ++ auto m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6][ib].qs), _mm_loadu_si128((const __m128i *)x8[2][ib].qs)); ++ auto m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7][ib].qs), _mm_loadu_si128((const __m128i *)x8[3][ib].qs)); ++ auto t0 = _mm256_unpacklo_epi32(m0, m1); ++ auto t1 = _mm256_unpacklo_epi32(m2, m3); ++ auto t2 = _mm256_unpackhi_epi32(m0, m1); ++ auto t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++ //#ifdef HAVE_FANCY_SIMD ++ // m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++ // m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++ // m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++ // m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++ //#endif ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 0, m0); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 1, m1); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 2, m2); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 3, m3); ++ m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[0][ib].qs+1)); ++ m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[1][ib].qs+1)); ++ m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[2][ib].qs+1)); ++ m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7][ib].qs+1), _mm_loadu_si128((const __m128i *)x8[3][ib].qs+1)); ++ t0 = _mm256_unpacklo_epi32(m0, m1); ++ t1 = _mm256_unpacklo_epi32(m2, m3); ++ t2 = _mm256_unpackhi_epi32(m0, m1); ++ t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++ //#ifdef HAVE_FANCY_SIMD ++ // m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++ // m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++ // m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++ // m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++ //#endif ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 4, m0); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 5, m1); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 6, m2); ++ _mm256_storeu_si256((__m256i *)y[ib].qs + 7, m3); ++#elif defined __ARM_NEON ++ for (int l = 0; l < 2; ++l) { ++ m0.val[0] = vld1q_s8(x8[0][ib].qs+16*l); m0.val[1] = vld1q_s8(x8[4][ib].qs+16*l); ++ m1.val[0] = vld1q_s8(x8[1][ib].qs+16*l); m1.val[1] = vld1q_s8(x8[5][ib].qs+16*l); ++ m2.val[0] = vld1q_s8(x8[2][ib].qs+16*l); m2.val[1] = vld1q_s8(x8[6][ib].qs+16*l); ++ m3.val[0] = vld1q_s8(x8[3][ib].qs+16*l); m3.val[1] = vld1q_s8(x8[7][ib].qs+16*l); ++ auto row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[0]), vreinterpretq_s32_s8(m1.val[0])); ++ auto row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[0]), vreinterpretq_s32_s8(m3.val[0])); ++ m0.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[1]), vreinterpretq_s32_s8(m1.val[1])); ++ row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[1]), vreinterpretq_s32_s8(m3.val[1])); ++ m0.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ vst1q_s8_x2(y[ib].qs + 0 + 128*l, m0); ++ vst1q_s8_x2(y[ib].qs + 32 + 128*l, m1); ++ vst1q_s8_x2(y[ib].qs + 64 + 128*l, m2); ++ vst1q_s8_x2(y[ib].qs + 96 + 128*l, m3); ++ } ++#else ++ for (int l = 0; l < 4; ++l) { ++ for (int k = 0; k < 8; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i+ 0] = x8[k][ib].qs[i+4*l+ 0]; ++ y[ib].qs[32*l+4*k+i+128] = x8[k][ib].qs[i+4*l+16]; ++ } ++ } ++#endif ++ } ++ y += nblock; ++ } ++ return result; ++} ++ ++#include "iqk_flash_impl.h" ++#include "fa/iqk_fa_templates.h" ++ ++bool iqk_flash_attn_impl(int int_type_k, // type of k ++ int int_type_v, // type of v ++ int Dk, // K head size ++ int Dv, // V head size ++ int nq1, // number of columns in q ++ int nk1, // number of rows in k ++ int stride_q, // distance between q columns in bytes ++ int stride_k, // distance between k rows in bytes ++ int stride_v, // distance between v rows in bytes ++ int stride_m, // distance between mask rows (in bytes ++ int stride_qkv, // distance between rows in mask (in bytes) ++ const float * q, // q matrix. ++ const void * k, // k matrix. Assumed to be fp16, nq x nk elements ++ const void * v, // v matrix. Assumed to be fp16, nq x nk elements ++ const void * mask, // mask. If not null, assumed to be fp16. nq x nk elements ++ const float * sinksf, // mask. If not null, assumed to be fp16. nq x nk elements ++ [[maybe_unused]] int nsinks, ++ float scale, // scale applied before softmax ++ float softcap, // if > 0, a "soft-cap" operation is applied before softmax ++ float * qkv, // v*softmax(scale*(k*q)) ++ float * M, float * S) { ++ ++ if (!mask || nk1%32 != 0) return false; // the implementation assumes mask is not null and nk is a multiple of 32 ++ ++ if (Dk == 576 && Dv == 512) { ++ return iqk_fa_576_512(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ if (Dk == 512 && Dv == 512) { ++ return iqk_fa_512_512(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ if (Dk == 320 && Dv == 256) { ++ return iqk_fa_320_256(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 192 && Dv == 128) { ++ return iqk_fa_192_128(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 192 && Dv == 192) { ++ return iqk_fa_192_192(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 256 && Dv == 256) { ++ return iqk_fa_256_256(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 128 && Dv == 128) { ++ return iqk_fa_128_128(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 96 && Dv == 96) { ++ return iqk_fa_96_96(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ if (Dk == 64 && Dv == 64) { ++ return iqk_fa_64_64(int_type_k, int_type_v, nq1, nk1, stride_q, stride_k, stride_v, stride_m, stride_qkv, ++ q, k, v, mask, scale, softcap, qkv, sinksf, M, S); ++ } ++ ++ return false; ++} ++#endif ++ ++#endif // IQK_IMPLEMENT +diff --git a/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp b/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp +@@ -0,0 +1,525 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#include "iqk_config.h" ++#include "iqk_mul_mat.h" ++#include "iqk_flash_impl.h" ++#include "ggml.h" ++#include "iqk_ggml_type_ext.h" // opencoti F5-opt W2 (#290): ik_llama ggml_type enum delta (this TU includes ggml.h directly, not the ggml-common.h shim) ++ ++#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace { ++inline uint32_t simple_gcd(uint32_t a, uint32_t b) { ++ while (a != b) { ++ if (a > b) a -= b; ++ else b -= a; ++ } ++ return a; ++} ++inline void accumulate_qkv(int Dv, float& M, float& S, float Mj, float Sj, float * Racc, const float * R) { ++ if (Mj == -INFINITY) return; ++ if (Mj > M) { ++ if (M == -INFINITY) { ++ std::memcpy(Racc, R, Dv*sizeof(float)); ++ S = Sj; ++ } else { ++ float c = exp(M - Mj); ++ S = c*S + Sj; ++ for (int i = 0; i < Dv; ++i) Racc[i] = c*Racc[i] + R[i]; ++ } ++ M = Mj; ++ } else { ++ float c = exp(Mj - M); ++ S += c*Sj; ++ for (int i = 0; i < Dv; ++i) Racc[i] += c*R[i]; ++ } ++} ++} ++ ++size_t iqk_fa_work_buffer_size(const struct ggml_tensor * dst, int nth) { ++ auto Q = dst->src[0]; ++ auto K = dst->src[1]; ++ auto V = dst->src[2]; ++ int rk2 = Q->ne[2]/K->ne[2]; ++ size_t size = 0; ++ if (Q->ne[1] >= 8 && K->type == GGML_TYPE_Q8_0) { ++ size = ggml_row_size(GGML_TYPE_Q8_0, K->ne[0]) * K->ne[1]*K->ne[2]*K->ne[3]; ++ } ++ if (Q->ne[1] == 1 && Q->ne[3] == 1 && Q->ne[2]/K->ne[2] > 1 && nth >= 1 && K->ne[1]/32 > 1) { ++ if (K->ne[2] > 1) { ++ int gcd = simple_gcd(K->ne[2], nth); ++ int nth_k = nth/gcd; ++ int nek2_k = K->ne[2]/gcd; ++ int nchunk = nek2_k*K->ne[1]/32; ++ int npt = (nchunk + nth_k - 1)/nth_k; ++ int nk; ++ if (npt*nth_k == nchunk) { ++ nk = 32 * (K->ne[1]*K->ne[2]/(32*nth)); ++ } else { ++ //int nm = std::max(1, npt/8); ++ int nm = 1; ++ while (true) { ++ if (nm*4 >= npt) break; ++ nm *= 2; ++ } ++ nk = 32*nm; ++ } ++ int nkk = (K->ne[1] + nk - 1)/nk; ++ int nstep_k = K->ne[2]*nkk; ++ size_t result_size = (V->ne[0] + 16)*Q->ne[2]/K->ne[2]*sizeof(float); ++ size += nstep_k*result_size; ++ return size; ++ } ++ int nstep_k = K->ne[1]/32; ++ if (nstep_k >= 4*nth) { ++ auto size_thread = (V->ne[0] + 16)*rk2*sizeof(float); ++ size += size_thread*nth; ++ return size; ++ } ++ int gcd_k = simple_gcd(nstep_k, nth); ++ if (gcd_k >= 1) { ++ int nth_k = nth/gcd_k; ++ int nq_per_thread = (rk2 + nth_k - 1)/nth_k; ++ if (nq_per_thread > 1) { ++ auto size_thread = (V->ne[0] + 16)*nq_per_thread*sizeof(float); ++ size += size_thread*nth; ++ return size; ++ } ++ } ++ int rv2 = Q->ne[2] / V->ne[2]; ++ if (Q->ne[1] == 1 && Q->ne[3] == 1 && rk2 > 1 && rk2 == rv2 && K->ne[1]*K->ne[2] >= 32*nth) { ++ auto result_size = (V->ne[0] + 16)*rk2*sizeof(float); ++ size += result_size*nth; ++ } ++ return size; ++ } ++ return size; ++} ++ ++static inline const std::unordered_set & supported_kv_types() { ++#ifdef GGML_IQK_FA_ALL_QUANTS ++ static std::unordered_set k_supported = { ++ GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q8_KV, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_IQ4_NL ++ }; ++#else ++ static std::unordered_set k_supported = { ++ GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q8_KV, GGML_TYPE_Q6_0, ++ }; ++#endif ++ return k_supported; ++} ++ ++static inline bool are_kv_types_supported(ggml_type type_k, ggml_type type_v) { ++ if (type_k == GGML_TYPE_BF16) { ++ if (type_v != type_k) { ++ return false; ++ } ++#ifdef __AVX512BF16__ ++ return true; ++#else ++ return false; ++#endif ++ } ++ auto & supported = supported_kv_types(); ++ auto it_k = supported.find(type_k); ++ auto it_v = supported.find(type_v); ++ return it_k != supported.end() && it_v != supported.end(); ++} ++ ++// TODO: get the ggml_type enum here without polution ++// ++extern "C" IQK_API bool iqk_flash_attn_noalibi(int type_q, int type_mask, float max_bias, ++ int neq3, int neq2, long nbq3, long nbq2, ++ int nek3, int nek2, long nbk3, long nbk2, ++ int nev3, int nev2, long nbv3, long nbv2, ++ int ne2, int ne1, long nb1, ++ int int_type_k_in, // type of k ++ int int_type_v, // type of v ++ int Dk, // K head size ++ int Dv, // V head size ++ int neq1, // number of columns in q ++ int nek1, // number of rows in k ++ int stride_q, // distance between q columns in bytes ++ int stride_k, // distance between k rows in bytes ++ int stride_v, // distance between v rows in bytes ++ int stride_m, // distance between mask rows (in bytes ++ const void * q, // q matrix. ++ const void * k, // k matrix. Assumed to be fp16, nq x nk elements ++ const void * v, // v matrix. Assumed to be fp16, nq x nk elements ++ const void * mask, // mask. If not null, assumed to be fp16. nq x nk elements ++ const void * sinks, // mask. If not null, assumed to be fp16. nq x nk elements ++ float scale, // scale applied before softmax ++ float softcap, // if > 0, a "soft-cap" operation is applied before softmax ++ float * qkv, // v*softmax(scale*(k*q)) ++ [[maybe_unused]] void * work_buffer_in, [[maybe_unused]] barrier_t barrier, [[maybe_unused]] void * barrier_data, ++ int ith, int nth, int n_swa) { ++ ++ if (type_q != 0 || type_mask != 1 || max_bias > 0) return false; ++ ++ if (auto type_k = ggml_type(int_type_k_in), type_v = ggml_type(int_type_v); !are_kv_types_supported(type_k, type_v)) { ++ if (ith == 0) { ++ fprintf(stderr, "\n==================== K cache %s coupled with V cache %s is not a supported combination on the CPU backend.\n", ++ ggml_type_name(type_k), ggml_type_name(type_v)); ++ auto & supported = supported_kv_types(); ++ fprintf(stderr, "Supported types are:\n"); ++ for (auto type : supported) { ++ fprintf(stderr, " %s\n", ggml_type_name(type)); ++ } ++ fprintf(stderr, " Warning: ik_llama.cpp does not support Q5_0 or Q5_1 KV cache on the CPU.\n"); ++#ifdef __AVX512BF16__ ++ fprintf(stderr, " %s, but only if K and V are both %s\n", ggml_type_name(GGML_TYPE_BF16), ggml_type_name(GGML_TYPE_BF16)); ++#endif ++#ifndef GGML_IQK_FA_ALL_QUANTS ++ fprintf(stderr, " To enable q4_0, q4_1, and iq4_nl KV cache types, recompile with -DGGML_IQK_FA_ALL_QUANTS=ON\n"); ++#endif ++ } ++ barrier(barrier_data); ++ GGML_ABORT("Fatal error"); ++ } ++ ++ if (n_swa > 0 && mask) { ++ constexpr int kMinBatch = 256; ++ int ntokens = std::max(kMinBatch, neq1); ++ int nblock = (ntokens + n_swa + kMinBatch - 1)/kMinBatch; ++ int first = nek1 - nblock*kMinBatch; ++ if (first > 0) { ++ k = (const char *)k + int64_t(first)*stride_k; ++ v = (const char *)v + int64_t(first)*stride_v; ++ mask = (const uint16_t *)mask + first; ++ nek1 -= first; ++ } ++ } ++ ++ int rk2 = neq2/nek2; ++ int rv2 = neq2/nev2; ++ int rk3 = neq3/nek3; ++ int rv3 = neq3/nev3; ++ ++ int first_k = 0, last_k = nek1; ++ if (neq3 == 1 && rk2 > 1 && neq1 == 1 && nek1 > 256 && mask) { ++ // This is a quick hack for SWA models. ++ // Given that the mask is the same for all layers, ideally we should determine the ++ // cache bounds once, and reuse for the whole graph. But even with this simple hack ++ // we get non-negligible performance gains for SWA models and long context. ++ auto umask = (const uint16_t *)mask; ++ for (; first_k < last_k; ++first_k) { ++ if (umask[first_k] == 0) break; ++ } ++ for (; last_k > first_k; --last_k) { ++ if (umask[last_k-1] == 0) break; ++ } ++ int non = 32*((last_k - first_k + 31)/32); ++ first_k = std::max(0, last_k - non); ++ last_k = std::min(first_k + non, nek1); ++ //printf("nek1 = %d, first = %d, last = %d\n", nek1, first, last); ++ if (last_k - first_k <= 3*nek1/4 && (last_k - first_k)%32 == 0) { ++ //printf("Reducing from %d to %d\n", nek1, last_k - first_k); ++ k = (const void *)((const char *)k + first_k*stride_k); ++ v = (const void *)((const char *)v + first_k*stride_v); ++ mask = (const void *)((const uint16_t *)mask + first_k); ++ nek1 = last_k - first_k; ++ } ++ } ++ ++ int int_type_k = int_type_k_in; ++ auto work_buffer = work_buffer_in; ++ if (neq1 >= 8) { ++ uint64_t row_size = 0; ++ work_buffer = iqk_repack_k(int_type_k, Dk, nek1, nek2, nek3, stride_k, nbk2, nbk3, k, work_buffer_in, ith, nth, int_type_k, row_size); ++ if (int_type_k != int_type_k_in) { ++ stride_k = row_size; ++ nbk2 = stride_k*nek1; ++ nbk3 = nbk2*nek2; ++ k = work_buffer_in; ++ barrier(barrier_data); ++ } ++ } ++ //uint64_t row_size = 0; ++ //auto work_buffer = iqk_repack_k(int_type_k, Dk, nek1, nek2, nek3, stride_k, nbk2, nbk3, k, work_buffer_in, ith, nth, int_type_k, row_size); ++ //if (int_type_k != int_type_k_in) { ++ // stride_k = row_size; ++ // nbk2 = stride_k*nek1; ++ // nbk3 = nbk2*nek2; ++ // k = work_buffer_in; ++ // barrier(barrier_data); ++ //} ++ ++ // Getting confused all the time about where to load data from and store the results to ++ // (especially when combining the results from the threads). ++ // So, for now, making it work just for MLA (nek2 = 1). ++ // I think it would also speed up things for GQA, but I'm leaving this for another day. ++ if (neq3 == 1 && rk2 > 1 && neq1 == 1 && nth >= 1 && nek1/32 > 1 && nek2 == 1) { ++ int nstep_k = nek1/32; ++ if (nstep_k >= 4*nth) { ++ int nstep_k_per_thread = (nstep_k + nth - 1)/nth; ++ int ith_mid = nth; ++ int nstep_k_this_thread = nstep_k_per_thread; ++ if (nstep_k_per_thread*nth > nstep_k) { ++ ith_mid = nstep_k - nth*(nstep_k_per_thread - 1); ++ if (ith >= ith_mid) --nstep_k_this_thread; ++ } ++ //if (ith == 0) fprintf(stderr, "nstep_k = %d, nstep_k_per_thread = %d, ith_mid = %d\n", nstep_k, nstep_k_per_thread, ith_mid); ++ nstep_k_per_thread *= 32; ++ nstep_k_this_thread *= 32; ++ ++ auto kv_offset = ith <= ith_mid ? ith*nstep_k_per_thread ++ : ith_mid*nstep_k_per_thread + (ith - ith_mid)*nstep_k_this_thread; ++ auto kth = (const char *)k + kv_offset*stride_k; ++ auto vth = (const char *)v + kv_offset*stride_v; ++ auto qth = (const char *)q; ++ auto mth = mask ? (const char *)mask + kv_offset*sizeof(uint16_t) : nullptr; // we don't have ggml_half available here ++ ++ auto work = (char *)work_buffer; ++ auto size_thread = (Dv + 16)*rk2*sizeof(float); ++ auto result_buffer = work; ++ auto work_this_thread = (float *)(result_buffer + ith*size_thread); ++ if (!iqk_flash_attn_impl(int_type_k, int_type_v, ++ Dk, Dv, rk2, nstep_k_this_thread, nbq2, stride_k, stride_v, 0, Dv, //Dk*sizeof(uint16_t), Dv, ++ (const float *)qth, (const void *)kth, (const void *)vth, (const void *)mth, nullptr, 0, ++ scale, softcap, ++ work_this_thread, work_this_thread + (Dv+0)*rk2, work_this_thread + (Dv+1)*rk2)) return false; ++ ++ barrier(barrier_data); ++ ++ for (int j = ith; j < rk2; j += nth) { ++ auto Racc = qkv + j*nb1/sizeof(float); ++ float M = -INFINITY, S = 0; ++ for (int jth = 0; jth < nth; ++jth) { ++ auto R = (const float *)(result_buffer + jth*size_thread); ++ auto Mj = R + Dv*rk2; ++ auto Sj = Mj + rk2; ++ R += j*Dv; ++ accumulate_qkv(Dv, M, S, Mj[j], Sj[j], Racc, R); ++ } ++ float norm = S > 0 ? 1/S : 1; ++ for (int i = 0; i < Dv; ++i) Racc[i] *= norm; ++ } ++ return true; ++ } ++ int gcd_k = simple_gcd(nstep_k, nth); ++ if (gcd_k >= 1) { ++ int nth_k = nth/gcd_k; ++ int ith_k = ith%gcd_k; ++ int ith_q = ith/gcd_k; ++ int nq_per_thread = (rk2 + nth_k - 1)/nth_k; ++ if (nq_per_thread > 1) { ++ int ith_mid = nth_k; ++ int nq_this_thread = nq_per_thread; ++ if (nq_per_thread*nth_k > rk2) { ++ ith_mid = rk2 - nth_k*(nq_per_thread - 1); ++ if (ith_q >= ith_mid) --nq_this_thread; ++ } ++ int j_mid = ith_mid*nq_per_thread; ++ auto work = (char *)work_buffer; ++ auto size_thread = (Dv + 16)*nq_per_thread*sizeof(float); ++ auto result_buffer = work; ++ ++ auto kth = (const char *)k + ith_k*(nek1/gcd_k)*stride_k; ++ auto vth = (const char *)v + ith_k*(nek1/gcd_k)*stride_v; ++ auto q_offset = ith_q < ith_mid ? ith_q*nq_per_thread*nbq2 : (ith_mid*nq_per_thread + (ith_q - ith_mid)*nq_this_thread)*nbq2; ++ auto qth = (const char *)q + q_offset; ++ auto mth = mask ? (const char *)mask + ith_k*(nek1/gcd_k)*sizeof(uint16_t) : nullptr; // we don't have ggml_half available here ++ ++ // Each thread will produce a result of size Dv*nq_this_thread*sizeof(float) ++ // In addition, we need M, S for the nq_this_thread rows the thread is processing ++ // => (Dv + 2)*nq_per_thread*sizeof(float). We use (Dv + 16) instead to make sure threads are not ++ // writing onto the same cache line. ++ auto work_this_thread = (float *)(result_buffer + ith*size_thread); ++ if (!iqk_flash_attn_impl(int_type_k, int_type_v, ++ Dk, Dv, nq_this_thread, nek1/gcd_k, nbq2, stride_k, stride_v, 0, Dv, //Dk*sizeof(uint16_t), Dv, ++ (const float *)qth, (const void *)kth, (const void *)vth, (const void *)mth, nullptr, 0, ++ scale, softcap, ++ work_this_thread, work_this_thread + (Dv+0)*nq_this_thread, work_this_thread + (Dv+1)*nq_this_thread)) return false; ++ ++ barrier(barrier_data); ++ ++ // There are nek1/gcd_k contributions for each j that we need to sum up ++ // Thread i computed k/v (i%gcd_k)*(nek1/gcd_k) for j (i/gcd_k)*(rk2/nth_k)...((i/gcd_k)+1)*(rk2/nth_k) and results at offset i*size_thread ++ ++ // TODO: simdify this ++ // TODO: if nth > rk2, have threads process portions of the rows instead of entire rows as it is now ++ for (int j = ith; j < rk2; j += nth) { ++ auto Racc = qkv + j*nb1/sizeof(float); ++ float M = -INFINITY, S = 0; ++ int jth_first, jj, nq_this_j; ++ if (j < j_mid) { ++ jth_first = j/nq_per_thread; ++ jj = j%nq_per_thread; ++ nq_this_j = nq_per_thread; ++ } else { ++ jth_first = ith_mid + (j - j_mid)/(nq_per_thread-1); ++ jj = (j - j_mid)%(nq_per_thread-1); ++ nq_this_j = nq_per_thread - 1; ++ } ++ jth_first *= gcd_k; ++ for (int jth = jth_first; jth < jth_first + gcd_k; ++jth) { ++ auto R = (const float *)(result_buffer + jth*size_thread); ++ auto Mj = R + Dv*nq_this_j; ++ auto Sj = Mj + nq_this_j; ++ R += jj*Dv; ++ accumulate_qkv(Dv, M, S, Mj[jj], Sj[jj], Racc, R); ++ } ++ float norm = S > 0 ? 1/S : 1; ++ for (int i = 0; i < Dv; ++i) Racc[i] *= norm; ++ } ++ return true; ++ } ++ } ++ } ++ ++ if (neq3 == 1 && rk2 > 1 && rk2 == rv2 && neq1 == 1 && nth >= 1 && nek2*nek1 >= 32*nth) { ++ auto result_size = (Dv + 16)*rk2*sizeof(float); ++ int gcd = simple_gcd(nek2, nth); ++ int nth_k = nth/gcd; ++ int nek2_k = nek2/gcd; ++ int nchunk = nek2_k*nek1/32; ++ int npt = (nchunk + nth_k - 1)/nth_k; ++ int nk; ++ if (npt*nth_k == nchunk) { ++ nk = 32 * (nek2*nek1/(32*nth)); ++ } else { ++ //int nm = std::max(1, npt/8); ++ int nm = 1; ++ while (true) { ++ if (nm*4 >= npt) break; ++ nm *= 2; ++ } ++ nk = 32*nm; ++ } ++ //int nk = 32 * (nek2*nek1/(32*nth)); ++ int nkk = (nek1 + nk - 1)/nk; ++ int nstep_k = nek2*nkk; ++ //if (ith == 0) printf("rk2 = %d, nek1 = %d, nek2 = %d, nk = %d, nkk = %d, nstep_k = %d\n", (int)rk2, (int)nek1, (int)nek2, nk, nkk, nstep_k); ++ for (int istep_k = ith; istep_k < nstep_k; istep_k += nth) { ++ int ik02 = istep_k/nkk; ++ int ik01 = nk*(istep_k - ik02*nkk); ++ int this_nk = ik01 + nk <= nek1 ? nk : nek1 - ik01; ++ if (this_nk <= 0) break; ++ auto this_result = (float *)((char *)work_buffer + istep_k*result_size); ++ auto this_q = (const float *)((const char *)q + ik02*rk2*nbq2); ++ auto this_k = (const char *)k + ik01*stride_k + ik02*nbk2; ++ auto this_v = (const char *)v + ik01*stride_v + ik02*nbv2; ++ auto this_m = mask ? (const char *)mask + ik01*sizeof(uint16_t) : nullptr; // we don't have ggml_half available here ++ if (!iqk_flash_attn_impl(int_type_k, int_type_v, ++ Dk, Dv, rk2, this_nk, nbq2, stride_k, stride_v, 0, Dv, ++ this_q, (const void *)this_k, (const void *)this_v, (const void *)this_m, nullptr, 0, ++ scale, softcap, this_result, this_result + (Dv+0)*rk2, this_result + (Dv+1)*rk2)) return false; ++ } ++ ++ barrier(barrier_data); ++ ++ // We have nkk results for each head ++ for (int iq2 = ith; iq2 < neq2; iq2 += nth) { ++ // ik02*rk2 + il = iq2 (il = 0...rk2-1) => ik02 = iq2/rk2, il = iq2%rk2; ++ int ik02 = iq2/rk2; ++ int il = iq2 - ik02*rk2; ++ auto Racc = qkv + iq2*nb1/sizeof(float); ++ //std::memset(Racc, 0, Dv*sizeof(float)); ++ float M = -INFINITY, S = 0; ++ for (int ikk = 0; ikk < nkk; ++ikk) { ++ int istep_k = ik02*nkk + ikk; ++ auto this_result = (float *)((char *)work_buffer + istep_k*result_size); ++ const float * R = this_result + il*Dv; ++ const float * Mj = this_result + Dv*rk2; ++ const float * Sj = Mj + rk2; ++ accumulate_qkv(Dv, M, S, Mj[il], Sj[il], Racc, R); ++ } ++ if (sinks) { ++ float s = ((const float *)sinks)[iq2]; ++ if (s > M) { ++ float m = expf(M - s); ++ for (int i = 0; i < Dv; ++i) Racc[i] *= m; ++ S = S*m + 1; ++ } else { ++ S += expf(s - M); ++ } ++ } ++ float norm = S > 0 ? 1/S : 1; ++ for (int i = 0; i < Dv; ++i) Racc[i] *= norm; ++ } ++ return true; ++ } ++ ++ // I keep changing my mind what is the best strategy to split the threads when processing ++ // multiple heads. This is my current thinking, the commented out code below was the previous. ++ int ntg = nth/simple_gcd(neq2*neq3, nth); ++ int neq1g = (neq1 + ntg - 1)/ntg; ++ //int64_t work_per_slice = D*nek1*neq1; ++ //int ntg = 1; ++ // ++ // When neq1 is large, it is better to have more than one thread process one (iq2,iq3) matrix ++ // But we also want each thread to process the same amount of rows, so neq1 must be a multiple of ++ // the number of threads processing the (iq2, iq3) matrix. ++ // ++ //if (neq1 >= 8*nth) { ++ // if (nth%8 == 0 && neq1%8 == 0 && work_per_slice >= (1 << 23)) ntg = 8; ++ // else if (nth%4 == 0 && neq1%4 == 0 && work_per_slice >= (1 << 21)) ntg = 4; ++ // else if (nth%2 == 0 && neq1%2 == 0 && work_per_slice >= (1 << 19)) ntg = 2; ++ //} ++ int counter = 0; ++ for (int64_t iq3 = 0; iq3 < neq3; iq3++) { ++ for (int64_t iq2 = 0; iq2 < neq2; iq2++) { ++ auto sinksf = sinks ? (const float *)sinks + iq2 : nullptr; ++ if (counter++ % (nth/ntg) == ith/ntg) { ++ int iq1 = (ith%ntg)*neq1g; ++ int this_neq1 = std::min(neq1g, neq1-iq1); ++ if (this_neq1 > 0) { ++ if (!iqk_flash_attn_impl(int_type_k, int_type_v, ++ Dk, Dv, this_neq1, nek1, stride_q, stride_k, stride_v, stride_m, ne1*nb1/sizeof(float), ++ (const float *)((const char *)q + iq2*nbq2 + iq3*nbq3 + iq1*stride_q), ++ (const void *)((const char *)k + iq2/rk2*nbk2 + iq3/rk3*nbk3), ++ (const void *)((const char *)v + iq2/rv2*nbv2 + iq3/rv3*nbv3), ++ mask ? (const void *)((const char *)mask + iq1*stride_m) : nullptr, sinksf, 1, ++ scale, softcap, ++ (float *)((char *)qkv + (iq3*ne2*ne1 + iq2 + iq1*ne1)*nb1), nullptr, nullptr)) return false; ++ } ++ } ++ } ++ } ++ ++ return true; ++} ++ ++#else ++ ++bool iqk_flash_attn_noalibi([[maybe_unused]] int type_q, [[maybe_unused]] int type_mask, [[maybe_unused]] float max_bias, ++ [[maybe_unused]] int neq3, [[maybe_unused]] int neq2, [[maybe_unused]] long nbq3, [[maybe_unused]] long nbq2, ++ [[maybe_unused]] int nek3, [[maybe_unused]] int nek2, [[maybe_unused]] long nbk3, [[maybe_unused]] long nbk2, ++ [[maybe_unused]] int nev3, [[maybe_unused]] int nev2, [[maybe_unused]] long nbv3, [[maybe_unused]] long nbv2, ++ [[maybe_unused]] int ne2, [[maybe_unused]] int ne1, [[maybe_unused]] long nb1, ++ [[maybe_unused]] int type_k, // type of k ++ [[maybe_unused]] int type_v, // type of v ++ [[maybe_unused]] int Dk, // K head size ++ [[maybe_unused]] int Dv, // V head size ++ [[maybe_unused]] int nq, // number of columns in q ++ [[maybe_unused]] int nk, // number of rows in k ++ [[maybe_unused]] int stride_q, // distance between q columns in bytes ++ [[maybe_unused]] int stride_k, // distance between k rows in bytes ++ [[maybe_unused]] int stride_v, // distance between v rows in bytes ++ [[maybe_unused]] int stride_m, // distance between mask rows (in bytes ++ [[maybe_unused]] const void * q, // q matrix. ++ [[maybe_unused]] const void * k, // k matrix. Assumed to be fp16, nq x nk elements ++ [[maybe_unused]] const void * v, // v matrix. Assumed to be fp16, nq x nk elements ++ [[maybe_unused]] const void * mask, // mask. If not null, assumed to be fp16. nq x nk elements ++ [[maybe_unused]] float scale, // scale applied before softmax ++ [[maybe_unused]] float softcap, // if > 0, a "soft-cap" operation is applied before softmax ++ [[maybe_unused]] float * qkv, // v*softmax(scale*(k*q)) ++ [[maybe_unused]] void * work_buffer, [[maybe_unused]] barrier_t barrier, [[maybe_unused]] void * barrier_data, ++ [[maybe_unused]] int ith, [[maybe_unused]] int nth, [[maybe_unused]] int n_swa) { ++ return false; ++} ++ ++#endif ++ +diff --git a/llama.cpp/ggml/src/iqk/iqk_flash_impl.h b/llama.cpp/ggml/src/iqk/iqk_flash_impl.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_flash_impl.h +@@ -0,0 +1,35 @@ ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++ ++#include ++ ++bool iqk_flash_attn_impl(int type_k, // type of k ++ int type_v, // type of v ++ int Dk, // K head size ++ int Dv, // V head size ++ int nq, // number of columns in q ++ int nk, // number of rows in k ++ int stride_q, // distance between q columns in bytes ++ int stride_k, // distance between k rows in bytes ++ int stride_v, // distance between v rows in bytes ++ int stride_m, // distance between mask rows (in bytes ++ int stride_qkv, // distance between rows in mask (in bytes) ++ const float * q, // q matrix. ++ const void * k, // k matrix. Assumed to be fp16, nq x nk elements ++ const void * v, // v matrix. Assumed to be fp16, nq x nk elements ++ const void * mask, // mask. If not null, assumed to be fp16. nq x nk elements ++ const float * sinksf, // attention sinks ++ int nsinks, // number of sinks ++ float scale, // scale applied before softmax ++ float softcap, // if > 0, a "soft-cap" operation is applied before softmax ++ float * qkv, // v*softmax(scale*(k*q)) ++ float * M, ++ float * S); ++ ++void * iqk_repack_k(int type_k, int nek0, int nek1, int nek2, int nek3, long nbk1, long nbk2, long nbk3, ++ const void * k, void * work, int ith, int nth, int& repacked_type, uint64_t& row_size); +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_floats.cpp b/llama.cpp/ggml/src/iqk/iqk_gemm_floats.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_floats.cpp +@@ -0,0 +1,1144 @@ ++#include "iqk_gemm_floats.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#include "ggml-impl.h" ++ ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++// float matrices - we handle f16, bf16 (if native bf16 support is available) and f32, but only to f32 result ++ ++struct QFBase { ++#ifdef __AVX512F__ ++ constexpr static int k_step = 16; ++ using Data = __m512; ++ using Acc = __m512; ++ static inline Data load(const ggml_half * x) { return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)x)); } ++ static inline Data load(const float * x) { return _mm512_loadu_ps(x); } ++ static inline Data load(const ggml_bf16_t * x) { ++ return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)x)), 16)); ++ } ++ static inline Acc acc(Acc prev, const Data& y, const Data& x) { ++ return _mm512_fmadd_ps(y, x, prev); ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return _mm512_mul_ps(y, x); ++ } ++ static inline Acc add(Acc x, Acc y) { return _mm512_add_ps(x, y); } ++ static inline float hsum(Acc acc) { ++ return _mm512_reduce_add_ps(acc); ++ } ++ template ++ static inline Data load4Floats(const Float * x) { ++ return _mm512_insertf32x4(_mm512_setzero_ps(), load128(x), 0); ++ } ++ static inline Acc acc_r4(Acc acc, const Data * xv, const Data& yv) { ++ acc = _mm512_fmadd_ps(xv[0], _mm512_shuffle_ps(yv, yv, 0x00), acc); ++ acc = _mm512_fmadd_ps(xv[1], _mm512_shuffle_ps(yv, yv, 0x55), acc); ++ acc = _mm512_fmadd_ps(xv[2], _mm512_shuffle_ps(yv, yv, 0xaa), acc); ++ acc = _mm512_fmadd_ps(xv[3], _mm512_shuffle_ps(yv, yv, 0xff), acc); ++ return acc; ++ } ++ static inline Acc acc_r4_first(const Data * xv, const Data& yv) { ++ auto acc = _mm512_mul_ps(xv[0], _mm512_shuffle_ps(yv, yv, 0x00)); ++ acc = _mm512_fmadd_ps(xv[1], _mm512_shuffle_ps(yv, yv, 0x55), acc); ++ acc = _mm512_fmadd_ps(xv[2], _mm512_shuffle_ps(yv, yv, 0xaa), acc); ++ acc = _mm512_fmadd_ps(xv[3], _mm512_shuffle_ps(yv, yv, 0xff), acc); ++ return acc; ++ } ++ static inline __m128 hsum_r4(Acc acc) { ++ auto sum1 = _mm_add_ps(_mm512_extractf32x4_ps(acc, 0), _mm512_extractf32x4_ps(acc, 1)); ++ auto sum2 = _mm_add_ps(_mm512_extractf32x4_ps(acc, 2), _mm512_extractf32x4_ps(acc, 3)); ++ return _mm_add_ps(sum1, sum2); ++ } ++#else ++ constexpr static int k_step = 8; ++ using Data = __m256; ++ using Acc = __m256; ++ static inline Data load(const ggml_half * x) { return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)x)); } ++ static inline Data load(const float * x) { return _mm256_loadu_ps(x); } ++ static inline Data load(const ggml_bf16_t * x) { ++ return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)x)), 16)); ++ } ++ static inline Acc acc(Acc prev, const Data& y, const Data& x) { ++ return _mm256_fmadd_ps(y, x, prev); ++ } ++ static inline Acc add(Acc x, Acc y) { return _mm256_add_ps(x, y); } ++ static inline Acc acc_r4(Acc acc, const Data * xv, const Data& yv) { ++ acc = _mm256_fmadd_ps(xv[0], _mm256_shuffle_ps(yv, yv, 0x00), acc); ++ acc = _mm256_fmadd_ps(xv[1], _mm256_shuffle_ps(yv, yv, 0x55), acc); ++ acc = _mm256_fmadd_ps(xv[2], _mm256_shuffle_ps(yv, yv, 0xaa), acc); ++ acc = _mm256_fmadd_ps(xv[3], _mm256_shuffle_ps(yv, yv, 0xff), acc); ++ return acc; ++ } ++ static inline Acc acc_r4_first(const Data * xv, const Data& yv) { ++ auto acc = _mm256_mul_ps(xv[0], _mm256_shuffle_ps(yv, yv, 0x00)); ++ acc = _mm256_fmadd_ps(xv[1], _mm256_shuffle_ps(yv, yv, 0x55), acc); ++ acc = _mm256_fmadd_ps(xv[2], _mm256_shuffle_ps(yv, yv, 0xaa), acc); ++ acc = _mm256_fmadd_ps(xv[3], _mm256_shuffle_ps(yv, yv, 0xff), acc); ++ return acc; ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return _mm256_mul_ps(y, x); ++ } ++ static inline float hsum(Acc acc) { ++ return hsum_float_8(acc); ++ } ++ static inline __m128 hsum_r4(Acc acc) { ++ return _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); ++ } ++ template ++ static inline Data load4Floats(const Float * x) { ++ return _mm256_insertf128_ps(_mm256_setzero_ps(), load128(x), 0); ++ } ++#endif ++ static inline __m128 load128(const ggml_half * x) { return _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)x)); } ++ static inline __m128 load128(const float * x) { return _mm_loadu_ps(x); } ++ static inline __m128 load128(const ggml_bf16_t * x) { ++ return _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i*)x)), 16)); ++ } ++}; ++ ++template struct QFT final : public QFBase { ++ constexpr static int nrc = nrc_in; ++ QFT(const DataInfo& info) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const Float *)info.src1_row(iy); ++ } ++ QFT(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const Float *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ IQK_ALWAYS_INLINE Data load_tail(int iy, int i) const { return load4Floats(y[iy] + 4*i); } ++ IQK_ALWAYS_INLINE void load_r4(int ix, int i, Data * xv) const { ++ xv[0] = load1(ix+0, i); ++ xv[1] = load1(ix+1, i); ++ xv[2] = load1(ix+2, i); ++ xv[3] = load1(ix+3, i); ++#ifdef __AVX512F__ ++ auto t0 = _mm512_unpacklo_ps(xv[0], xv[1]); ++ auto t1 = _mm512_unpacklo_ps(xv[2], xv[3]); ++ auto t2 = _mm512_unpackhi_ps(xv[0], xv[1]); ++ auto t3 = _mm512_unpackhi_ps(xv[2], xv[3]); ++ xv[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(t0), _mm512_castps_pd(t1))); ++ xv[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(t0), _mm512_castps_pd(t1))); ++ xv[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(t2), _mm512_castps_pd(t3))); ++ xv[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(t2), _mm512_castps_pd(t3))); ++#else ++ auto t0 = _mm256_unpacklo_ps(xv[0], xv[1]); ++ auto t1 = _mm256_unpacklo_ps(xv[2], xv[3]); ++ auto t2 = _mm256_unpackhi_ps(xv[0], xv[1]); ++ auto t3 = _mm256_unpackhi_ps(xv[2], xv[3]); ++ xv[0] = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(t0), _mm256_castps_pd(t1))); ++ xv[1] = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(t0), _mm256_castps_pd(t1))); ++ xv[2] = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(t2), _mm256_castps_pd(t3))); ++ xv[3] = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(t2), _mm256_castps_pd(t3))); ++#endif ++ } ++ const Float * y[nrc]; ++}; ++ ++// TBD if we want this ++//template ++//IQK_NOINLINE void mul_mat_Qx_Qy_Mx1(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++// static_assert(Qy::nrc == 1); ++// int nb = n/QFBase::k_step; ++// int nb4 = n/4; ++// Qy y(info); ++// Qx x(cx + ix0*bx, bx); ++// QFBase::Data xv[2*Qx::nrc]; ++// QFBase::Acc acc[2*Qx::nrc]; ++// auto yv1 = y.load1(0, 0); ++// auto yv2 = y.load1(0, 1); ++// for (int ix = 0; ix < Qx::nrc; ++ix) { ++// xv[2*ix+0] = x.load1(ix, 0); ++// xv[2*ix+1] = x.load1(ix, 1); ++// acc[2*ix+0] = QFBase::acc_first(yv1, xv[2*ix+0]); ++// acc[2*ix+1] = QFBase::acc_first(yv2, xv[2*ix+1]); ++// } ++// for (int i = 1; i < nb/2; ++i) { ++// yv1 = y.load1(0, 2*i+0); ++// yv2 = y.load1(0, 2*i+1); ++// for (int ix = 0; ix < Qx::nrc; ++ix) { ++// xv[2*ix+0] = x.load1(ix, 2*i+0); ++// xv[2*ix+1] = x.load1(ix, 2*i+1); ++// acc[2*ix+0] = QFBase::acc(acc[2*ix+0], yv1, xv[2*ix+0]); ++// acc[2*ix+1] = QFBase::acc(acc[2*ix+1], yv2, xv[2*ix+1]); ++// } ++// } ++// for (int i = (QFBase::k_step/4)*nb; i < nb4; ++i) { ++// yv1 = y.load_tail(0, i); ++// for (int ix = 0; ix < Qx::nrc; ++ix) { ++// xv[ix] = x.load_tail(ix, i); ++// acc[2*ix+0] = QFBase::acc(acc[2*ix+0], yv1, xv[ix]); ++// } ++// } ++// for (int ix = 0; ix < Qx::nrc; ++ix) info.store(ix0+ix, 0, QFBase::hsum(QFBase::add(acc[2*ix+0], acc[2*ix+1]))); ++//} ++ ++template ++IQK_NOINLINE void mul_mat_Qx_Qy_MxN(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ int nb = n/QFBase::k_step; ++ int nb4 = n/4; ++ Qy y(info); ++ Qx x(cx + ix0*bx, bx); ++ QFBase::Data xv[Qx::nrc]; ++ QFBase::Acc acc[Qx::nrc*Qy::nrc]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < Qx::nrc; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = QFBase::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < Qx::nrc; ++ix) acc[Qx::nrc*iy + ix] = QFBase::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = QFBase::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) acc[Qx::nrc*iy + ix] = QFBase::acc(acc[Qx::nrc*iy + ix], yv, xv[ix]); ++ } ++ } ++ for (int i = (QFBase::k_step/4)*nb; i < nb4; ++i) { ++ yv = y.load_tail(0, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) { ++ xv[ix] = x.load_tail(ix, i); ++ acc[ix] = QFBase::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc; ++iy) { ++ yv = y.load_tail(iy, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) acc[Qx::nrc*iy + ix] = QFBase::acc(acc[Qx::nrc*iy + ix], yv, xv[ix]); ++ } ++ } ++ for (int iy = 0; iy < Qy::nrc; ++iy) for (int ix = 0; ix < Qx::nrc; ++ix) info.store(ix0+ix, iy, QFBase::hsum(acc[Qx::nrc*iy+ix])); ++} ++ ++template ++inline void mul_mat_Qx_Qy_MxN_fa(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ int nb = n/QFBase::k_step; ++ Qy y(info); ++ Qx x(cx + ix0*bx, bx); ++ QFBase::Data xv[Qx::nrc]; ++ QFBase::Acc acc[Qx::nrc*Qy::nrc]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < Qx::nrc; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = QFBase::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < Qx::nrc; ++ix) acc[Qx::nrc*iy + ix] = QFBase::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = QFBase::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < Qx::nrc; ++ix) acc[Qx::nrc*iy + ix] = QFBase::acc(acc[Qx::nrc*iy + ix], yv, xv[ix]); ++ } ++ } ++ for (int iy = 0; iy < Qy::nrc; ++iy) for (int ix = 0; ix < Qx::nrc; ++ix) info.store(ix0+ix, iy, QFBase::hsum(acc[Qx::nrc*iy+ix])); ++} ++ ++template ++inline void mul_mat_Qx_Qy_MxN_fa4(int D, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ static_assert(Qx::nrc%4 == 0); ++ int nb = D/QFBase::k_step; ++ Qy y(info); ++ Qx x(cx + ix0*bx, bx); ++ QFBase::Data xv[Qx::nrc]; ++ QFBase::Acc acc[Qx::nrc*Qy::nrc/4] = {}; ++ for (int i = 0; i < nb; ++i) { ++ for (int ix = 0; ix < Qx::nrc/4; ++ix) x.load_r4(4*ix, i, xv + 4*ix); ++ for (int iy = 0; iy < Qy::nrc; ++iy) { ++ auto yv = y.load1(iy, i); ++ for (int ix = 0; ix < Qx::nrc/4; ++ix) acc[ix*Qy::nrc + iy] = QFBase::acc_r4(acc[ix*Qy::nrc + iy], xv + 4*ix, yv); ++ } ++ } ++ for (int iy = 0; iy < Qy::nrc; ++iy) { ++ for (int ix = 0; ix < Qx::nrc/4; ++ix) info.store(ix0+4*ix, iy, QFBase::hsum_r4(acc[ix*Qy::nrc + iy])); ++ } ++} ++ ++// This will handle any of f16 x f32, f32 x f16, f16 x f16, f32 x f32, with computations done ++// in f32 (i.e., f16 is first converted to f32). It is easy to extend to computations done in ++// f16, but I don't have a CPU capable of f16 vector arithmetic, so not doing it for now. ++template ++void mul_mat_fX_fY_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ const char * cx = (const char *)vx; ++ // TBD if we want this ++ //if constexpr (nrc_y == 1) { ++ // constexpr int k_nx = 2; ++ // for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ // mul_mat_Qx_Qy_Mx1, QFT>(n, cx, bx, ix*k_nx, info); ++ // } ++ // if (int lastx = k_nx*(nrc_x/k_nx); lastx < nrc_x) { ++ // int nx = nrc_x - lastx; ++ // switch (nx) { ++ // case 1: mul_mat_Qx_Qy_Mx1, QFT>(n, cx, bx, lastx, info); break; ++ // case 2: mul_mat_Qx_Qy_Mx1, QFT>(n, cx, bx, lastx, info); break; ++ // case 3: mul_mat_Qx_Qy_Mx1, QFT>(n, cx, bx, lastx, info); break; ++ // } ++ // //mul_mat_Qx_Qy_Mx1, QFT>(n, cx, bx, lastx, info); ++ // } ++ // return; ++ //} ++#ifdef __AVX512F__ ++ constexpr int k_nx = 5; ++#else ++ constexpr int k_nx = nrc_y == 1 ? 4 : 2; ++#endif ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++#ifdef __AVX512F__ ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ } ++#else ++ if constexpr (nrc_y == 1) { ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ } ++ } else { ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_MxN, QFT>(n, cx, bx, last_x, info); break; ++ } ++ } ++#endif ++} ++ ++#ifdef __AVX512BF16__ ++template ++static void mul_mat_bf16_r16_bf16(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%16 == 0); ++ const ggml_bf16_t * y[nrc_y]; ++ static_for([&](const int iy) { y[iy] = (const ggml_bf16_t *)info.src1_row(iy); }); ++ ++ for (int ix = 0; ix < nrc_x/32; ++ix) { ++ __m512 acc[2*nrc_y] = {}; ++ __m512bh qx[8]; ++ const ggml_bf16_t * b8_1 = (const ggml_bf16_t *)((const char *)vx + (32*ix+ 0)*bx); ++ const ggml_bf16_t * b8_2 = (const ggml_bf16_t *)((const char *)vx + (32*ix+16)*bx); ++ for (int ib = 0; ib < n/8; ++ib) { ++ qx[0] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_1+4*ib+0); ++ qx[1] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_1+4*ib+1); ++ qx[2] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_1+4*ib+2); ++ qx[3] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_1+4*ib+3); ++ qx[4] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_2+4*ib+0); ++ qx[5] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_2+4*ib+1); ++ qx[6] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_2+4*ib+2); ++ qx[7] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8_2+4*ib+3); ++ static_for([&](const int iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)y[iy]+ib); ++ //auto y = _mm512_broadcast_i32x4(y128); ++ auto y256 = MM256_SET_M128I(y128, y128); ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); ++ acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[0], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[1], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[2], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[3], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ acc[2*iy+1] = _mm512_dpbf16_ps(acc[2*iy+1], qx[4], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ acc[2*iy+1] = _mm512_dpbf16_ps(acc[2*iy+1], qx[5], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ acc[2*iy+1] = _mm512_dpbf16_ps(acc[2*iy+1], qx[6], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ acc[2*iy+1] = _mm512_dpbf16_ps(acc[2*iy+1], qx[7], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ }); ++ } ++ static_for([&](const int iy) { ++ info.store(32*ix+ 0, iy, acc[2*iy+0]); ++ info.store(32*ix+16, iy, acc[2*iy+1]); ++ }); ++ } ++ for (int ix = 32*(nrc_x/32); ix < nrc_x; ix += 16) { ++ __m512 acc[nrc_y] = {}; ++ __m512bh qx[4]; ++ const ggml_bf16_t * b8 = (const ggml_bf16_t *)((const char *)vx + (ix+0)*bx); ++ for (int ib = 0; ib < n/8; ++ib) { ++ qx[0] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8+4*ib+0); ++ qx[1] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8+4*ib+1); ++ qx[2] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8+4*ib+2); ++ qx[3] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8+4*ib+3); ++ static_for([&](const int iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)y[iy]+ib); ++ auto y256 = MM256_SET_M128I(y128, y128); ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); ++ acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[0], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[1], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[2], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[3], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ }); ++ } ++ static_for([&](const int iy) { ++ info.store(ix, iy, acc[iy]); ++ }); ++ } ++} ++ ++struct QFBaseBF16 { ++ constexpr static int k_step = 32; ++ using Data = __m512bh; ++ using Acc = __m512; ++ static inline Data load(const ggml_bf16_t * x) { return __m512bh(_mm512_loadu_si512((const __m512i *)x)); } ++ static inline Acc acc(Acc prev, Data y, Data x) { ++ return _mm512_dpbf16_ps(prev, y, x); ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return _mm512_dpbf16_ps(_mm512_setzero_ps(), y, x); ++ } ++ static inline float hsum(Acc acc) { ++ return _mm512_reduce_add_ps(acc); ++ } ++}; ++template struct QFTBF16 final : public QFBaseBF16 { ++ constexpr static int nrc = nrc_in; ++ QFTBF16(const DataInfo& info) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const ggml_bf16_t *)info.src1_row(iy); ++ } ++ QFTBF16(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const ggml_bf16_t *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ const ggml_bf16_t * y[nrc]; ++}; ++struct QFBaseBF16x8 { ++ constexpr static int k_step = 16; ++ using Data = __m256bh; ++ using Acc = __m256; ++ static inline Data load(const ggml_bf16_t * x) { return __m256bh(_mm256_loadu_si256((const __m256i *)x)); } ++ static inline Acc acc(Acc prev, Data y, Data x) { ++ return _mm256_dpbf16_ps(prev, y, x); ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return _mm256_dpbf16_ps(_mm256_setzero_ps(), y, x); ++ } ++ static inline float hsum(Acc acc) { ++ return hsum_float_8(acc); ++ } ++}; ++template struct QFTBF16x8 final : public QFBaseBF16x8 { ++ constexpr static int nrc = nrc_in; ++ QFTBF16x8(const DataInfo& info) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const ggml_bf16_t *)info.src1_row(iy); ++ } ++ QFTBF16x8(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc; ++iy) y[iy] = (const ggml_bf16_t *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ const ggml_bf16_t * y[nrc]; ++}; ++ ++template ++IQK_NOINLINE void mul_mat_Qx_Qy_MxN(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ int nb = n/QFBaseBF16::k_step; ++ QFTBF16 y(info); ++ QFTBF16 x(cx + ix0*bx, bx); ++ QFBaseBF16::Data xv[nrc_x]; ++ QFBaseBF16::Acc acc[nrc_x*nrc_y]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = QFBaseBF16::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QFBaseBF16::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = QFBaseBF16::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QFBaseBF16::acc(acc[nrc_x*iy + ix], yv, xv[ix]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) for (int ix = 0; ix < nrc_x; ++ix) info.store(ix0+ix, iy, QFBaseBF16::hsum(acc[nrc_x*iy+ix])); ++} ++ ++template ++void mul_mat_fX_fY_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ constexpr int k_nx = nrc_y <= 2 ? 8 : 5; ++ const char * cx = (const char *)vx; ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_Qx_Qy_MxN(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ if constexpr (nrc_y <= 2) { ++ if (nx >= 4) { ++ mul_mat_Qx_Qy_MxN(n, cx, bx, last_x, info); ++ last_x += 4; ++ if (last_x == nrc_x) return; ++ nx = nrc_x - last_x; ++ } ++ } ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_MxN(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_MxN(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_MxN(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_Qx_Qy_MxN(n, cx, bx, last_x, info); break; ++ } ++} ++template ++IQK_NOINLINE void mul_mat_Qx_Qy_MxNx8(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ int nb = n/QFBaseBF16x8::k_step; ++ QFTBF16x8 y(info); ++ QFTBF16x8 x(cx + ix0*bx, bx); ++ QFBaseBF16x8::Data xv[nrc_x]; ++ QFBaseBF16x8::Acc acc[nrc_x*nrc_y]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = QFBaseBF16x8::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QFBaseBF16x8::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = QFBaseBF16x8::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QFBaseBF16x8::acc(acc[nrc_x*iy + ix], yv, xv[ix]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) for (int ix = 0; ix < nrc_x; ++ix) info.store(ix0+ix, iy, QFBaseBF16x8::hsum(acc[nrc_x*iy+ix])); ++} ++ ++template ++void mul_mat_fX_fY_Tx8(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ constexpr int k_nx = nrc_y <= 2 ? 8 : 5; ++ const char * cx = (const char *)vx; ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_Qx_Qy_MxNx8(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ if constexpr (nrc_y <= 2) { ++ if (nx >= 4) { ++ mul_mat_Qx_Qy_MxNx8(n, cx, bx, last_x, info); ++ last_x += 4; ++ if (last_x == nrc_x) return; ++ nx = nrc_x - last_x; ++ } ++ } ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_MxNx8(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_MxNx8(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_MxNx8(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_Qx_Qy_MxNx8(n, cx, bx, last_x, info); break; ++ } ++} ++#endif ++ ++ ++template ++void set_mul_mat_f(std::array& funcs) { ++ for (auto& f : funcs) f = nullptr; ++ funcs[0] = mul_mat_fX_fY_T<1, FloatX, FloatY>; ++ funcs[1] = mul_mat_fX_fY_T<2, FloatX, FloatY>; ++ funcs[2] = mul_mat_fX_fY_T<3, FloatX, FloatY>; ++ funcs[3] = mul_mat_fX_fY_T<4, FloatX, FloatY>; ++ funcs[4] = mul_mat_fX_fY_T<5, FloatX, FloatY>; ++#ifndef __AVX512F__ ++ funcs[5] = mul_mat_fX_fY_T<6, FloatX, FloatY>; ++#endif ++} ++ ++#ifdef __AVX512BF16__ ++void set_mul_mat_bf16(std::array& funcs) { ++ for (auto& f : funcs) f = nullptr; ++ funcs[0] = mul_mat_fX_fY_T<1>; ++ funcs[1] = mul_mat_fX_fY_T<2>; ++ funcs[2] = mul_mat_fX_fY_T<3>; ++ funcs[3] = mul_mat_fX_fY_T<4>; ++ funcs[4] = mul_mat_fX_fY_T<5>; ++} ++void set_mul_mat_bf16x8(std::array& funcs) { ++ for (auto& f : funcs) f = nullptr; ++ funcs[0] = mul_mat_fX_fY_Tx8<1>; ++ funcs[1] = mul_mat_fX_fY_Tx8<2>; ++ funcs[2] = mul_mat_fX_fY_Tx8<3>; ++ funcs[3] = mul_mat_fX_fY_Tx8<4>; ++ funcs[4] = mul_mat_fX_fY_Tx8<5>; ++} ++void set_mul_mat_bf16_r16(std::array& funcs) { ++ for (auto& f : funcs) f = nullptr; ++ funcs[0] = mul_mat_bf16_r16_bf16<1>; ++ funcs[1] = mul_mat_bf16_r16_bf16<2>; ++ funcs[2] = mul_mat_bf16_r16_bf16<3>; ++ funcs[3] = mul_mat_bf16_r16_bf16<4>; ++ funcs[4] = mul_mat_bf16_r16_bf16<5>; ++ funcs[5] = mul_mat_bf16_r16_bf16<6>; ++ funcs[6] = mul_mat_bf16_r16_bf16<7>; ++ funcs[7] = mul_mat_bf16_r16_bf16<8>; ++} ++#endif ++ ++} // namespace ++ ++bool iqk_set_kernels_float(int ne00, int typeA, int typeB, std::array& kernels) { ++ ++ if (typeA == GGML_TYPE_BF16) { ++ if (ne00 % 8) return false; ++ switch (typeB) { ++#ifdef __AVX512BF16__ ++ case GGML_TYPE_BF16: { ++ if (ne00 % 16 == 0) { ++ set_mul_mat_bf16(kernels); ++ } else { ++ set_mul_mat_bf16x8(kernels); ++ } ++ } break; ++#else ++ case GGML_TYPE_BF16: set_mul_mat_f(kernels); break; ++ case GGML_TYPE_F32: set_mul_mat_f(kernels); break; ++#endif ++ default: return false; ++ } ++ return true; ++ } ++ ++ if (typeA == GGML_TYPE_BF16_R16) { ++ if (ne00 % 16) return false; ++ switch (typeB) { ++#ifdef __AVX512BF16__ ++ case GGML_TYPE_BF16: set_mul_mat_bf16_r16(kernels); break; ++#endif ++ default: return false; ++ } ++ return true; ++ } ++ ++ if (typeA == GGML_TYPE_F16 || typeA == GGML_TYPE_F32) { ++ if (ne00 % 4) return false; ++ } ++ if (typeA == GGML_TYPE_F16) { ++ switch (typeB) { ++ case GGML_TYPE_F16: set_mul_mat_f(kernels); break; ++ case GGML_TYPE_F32: set_mul_mat_f(kernels); break; ++ default: return false; ++ } ++ return true; ++ } ++ if (typeA == GGML_TYPE_F32) { ++ switch (typeB) { ++ case GGML_TYPE_F16: set_mul_mat_f(kernels); break; ++ case GGML_TYPE_F32: set_mul_mat_f(kernels); break; ++ default: return false; ++ } ++ return true; ++ } ++ ++ return false; ++ ++} ++ ++void iqk_gemm_default_floats(int D, int nq, const char * cx, size_t bx, DataInfo& info, int k_step) { ++ using q_float = float; ++#ifdef HAVE_FANCY_SIMD ++ constexpr int nrc_q = 8; ++ constexpr int nrc_k = 8; ++#else ++ // somewhat surprisingly, nrc_q = 4, nrc_k = 8 is better than nrc_q = 8, nrc_k = 4 ++ constexpr int nrc_q = 4; ++ constexpr int nrc_k = 8; ++#endif ++ GGML_ASSERT(k_step%nrc_k == 0); ++ int qrem = nq - nrc_q*(nq/nrc_q); ++ for (int iq = 0; iq < nq/nrc_q; ++iq) { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ info.cur_y += nrc_q; ++ } ++ if (qrem > 0) { ++ switch (qrem) { ++ case 1: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++ case 2: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++ case 3: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++#ifdef HAVE_FANCY_SIMD ++ case 4: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++ case 5: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++ case 6: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++ case 7: { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_Qx_Qy_MxN_fa4, QFT>(D, cx, bx, ik*nrc_k, info); ++ } ++ } break; ++#endif ++ } ++ } ++} ++ ++#else ++// ----------------------------------- __aarch64__ ----------------------------------------------- ++ ++namespace { ++ ++struct QF16Base { ++ constexpr static int k_step = 8; ++ using Data = float16x8_t; ++ using Acc = float16x8_t; ++ static inline Data load(const __fp16 * x) { return vld1q_f16(x); } ++ static inline Data load4(const __fp16 * x) { return vcombine_f16(vld1_f16(x), vdup_n_f16(0)); } ++ static inline Acc acc(Acc prev, const Data& y, const Data& x) { ++ return vfmaq_f16(prev, y, x); ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return vmulq_f16(y, x); ++ } ++ //constexpr static int k_step = 16; ++ //using Data = float16x8x2_t; ++ //static inline Data load(const __fp16 * x) { return vld1q_f16_x2(x); } ++ //static inline Acc acc(Acc prev, const Data& y, const Data& x) { ++ // return vfmaq_f16(vfmaq_f16(prev, y.val[0], x.val[0]), y.val[1], x.val[1]); ++ //} ++ //static inline Acc acc_first(const Data& y, const Data& x) { ++ // return vfmaq_f16(vmulq_f16(y.val[0], x.val[0]), y.val[1], x.val[1]); ++ //} ++ static inline float hsum(Acc acc) { ++ float32x4_t sum = vcvt_f32_f16(vadd_f16(vget_low_f16(acc), vget_high_f16(acc))); ++ return vaddvq_f32(sum); ++ } ++}; ++template struct QF16 final : public QF16Base { ++ using Base = QF16Base; ++ constexpr static int nrc_y = nrc; ++ QF16(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const __fp16 *)info.src1_row(iy); ++ } ++ QF16(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const __fp16 *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ IQK_ALWAYS_INLINE Data load_tail(int iy, int i) const { return load4(y[iy] + 4*i); } ++ IQK_ALWAYS_INLINE float16x8x4_t loadx(int iy, int i) const { return vld1q_f16_x4(y[iy] + 4*k_step*i); } ++ const __fp16 * y[nrc_y]; ++}; ++ ++struct QBF16Base { ++ constexpr static int k_step = 4; ++ using Data = float32x4_t; ++ using Acc = float32x4_t; ++ static inline Data load(const uint16_t * x) { return vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vld1_u16(x)), 16)); } ++ static inline Data load4(const uint16_t * x) { return load(x); } ++ static inline Acc acc(Acc prev, const Data& y, const Data& x) { ++ return vfmaq_f32(prev, y, x); ++ } ++ static inline Acc acc_first(const Data& y, const Data& x) { ++ return vmulq_f32(y, x); ++ } ++ static inline float hsum(Acc acc) { return vaddvq_f32(acc); } ++}; ++template struct QBF16 final : public QBF16Base { ++ using Base = QBF16Base; ++ constexpr static int nrc_y = nrc; ++ QBF16(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const uint16_t *)info.src1_row(iy); ++ } ++ QBF16(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const uint16_t *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ IQK_ALWAYS_INLINE Data load_tail(int iy, int i) const { return load(y[iy] + 4*i); } ++ const uint16_t * y[nrc_y]; ++}; ++ ++struct QF32Base { ++ constexpr static int k_step = 4; ++ using Data = float32x4_t; ++ using Acc = float32x4_t; ++ static inline Data load(const float * x) { return vld1q_f32(x); } ++ static inline Data load4(const float * x) { return load(x); } ++ static inline Acc acc(Acc prev, const Data& y, const Data& x) { return vfmaq_f32(prev, y, x); } ++ static inline Acc acc_first(const Data& y, const Data& x) { return vmulq_f32(y, x); } ++ static inline float hsum(Acc acc) { return vaddvq_f32(acc); } ++}; ++template struct QF32 final : public QF32Base { ++ using Base = QF32Base; ++ constexpr static int nrc_y = nrc; ++ QF32(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const float *)info.src1_row(iy); ++ } ++ QF32(const char * cx, size_t bx) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const float *)(cx + iy*bx); ++ } ++ IQK_ALWAYS_INLINE Data load1(int iy, int i) const { return load(y[iy] + k_step*i); } ++ IQK_ALWAYS_INLINE Data load_tail(int iy, int i) const { return load(y[iy] + 4*i); } ++ const float * y[nrc_y]; ++}; ++ ++template ++IQK_NOINLINE void mul_mat_Qx_Qy_NxN(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ GGML_ASSERT(Qx::Base::k_step == Qy::Base::k_step); ++ int nb = n/Qx::Base::k_step; ++ Qy y(info); ++ Qx x(cx + ix0*bx, bx); ++ typename Qx::Base::Data xv[Qx::nrc_y]; ++ typename Qx::Base::Acc acc[Qx::nrc_y*Qy::nrc_y]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = Qx::Base::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc_y; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) acc[Qx::nrc_y*iy + ix] = Qx::Base::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = Qx::Base::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc_y; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) acc[Qx::nrc_y*iy + ix] = Qx::Base::acc(acc[Qx::nrc_y*iy + ix], yv, xv[ix]); ++ } ++ } ++ if constexpr (Qx::Base::k_step > 4 && !is_multiple_of_k_step) { ++ int nb4 = n/4; ++ for (int i = (Qx::Base::k_step/4)*nb; i < nb4; ++i) { ++ yv = y.load_tail(0, i); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) { ++ xv[ix] = x.load_tail(ix, i); ++ acc[ix] = Qx::Base::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < Qy::nrc_y; ++iy) { ++ yv = y.load_tail(iy, i); ++ for (int ix = 0; ix < Qx::nrc_y; ++ix) acc[Qx::nrc_y*iy + ix] = Qx::Base::acc(acc[Qx::nrc_y*iy + ix], yv, xv[ix]); ++ } ++ } ++ } ++ for (int iy = 0; iy < Qy::nrc_y; ++iy) for (int ix = 0; ix < Qx::nrc_y; ++ix) info.store(ix0+ix, iy, Qx::Base::hsum(acc[Qx::nrc_y*iy+ix])); ++} ++ ++template ++IQK_NOINLINE void mul_mat_f16_f16_NxN(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ assert(n%QF16Base::k_step == 0); ++ int nb = n/QF16Base::k_step; ++ QF16 y(info); ++ QF16 x(cx + ix0*bx, bx); ++ QF16Base::Data xv[nrc_x]; ++ QF16Base::Acc acc[nrc_x*nrc_y]; ++ auto yv = y.load1(0, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, 0); ++ acc[ix] = QF16Base::acc_first(yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QF16Base::acc_first(yv, xv[ix]); ++ } ++ for (int i = 1; i < nb; ++i) { ++ yv = y.load1(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load1(ix, i); ++ acc[ix] = QF16Base::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load1(iy, i); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QF16Base::acc(acc[nrc_x*iy + ix], yv, xv[ix]); ++ } ++ } ++ if constexpr (!is_multiple_of_k_step) { ++ int nb4 = n/4; ++ for (int i = (QF16Base::k_step/4)*nb; i < nb4; ++i) { ++ yv = y.load_tail(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ xv[ix] = x.load_tail(ix, i); ++ acc[ix] = QF16Base::acc(acc[ix], yv, xv[ix]); ++ } ++ for (int iy = 1; iy < nrc_y; ++iy) { ++ yv = y.load_tail(iy, i); ++ for (int ix = 0; ix < nrc_x; ++ix) acc[nrc_x*iy + ix] = QF16Base::acc(acc[nrc_x*iy + ix], yv, xv[ix]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) for (int ix = 0; ix < nrc_x; ++ix) info.store(ix0+ix, iy, QF16Base::hsum(acc[nrc_x*iy+ix])); ++} ++ ++template typename Qx> ++void mul_mat_Qx_Qy_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%4 == 0); ++ constexpr int k_nx = 5; ++ const char * cx = (const char *)vx; ++ if (n%Qx::Base::k_step == 0) { ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_Qx_Qy_NxN, true>(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_NxN, true>(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_NxN, true>(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_NxN, true>(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_Qx_Qy_NxN, true>(n, cx, bx, last_x, info); break; ++ } ++ } else { ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_Qx_Qy_NxN, false>(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ switch (nx) { ++ case 1: mul_mat_Qx_Qy_NxN, false>(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_Qx_Qy_NxN, false>(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_Qx_Qy_NxN, false>(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_Qx_Qy_NxN, false>(n, cx, bx, last_x, info); break; ++ } ++ } ++} ++ ++template ++void mul_mat_f16_f16_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%4 == 0); ++ constexpr int k_nx = 5; ++ const char * cx = (const char *)vx; ++ if (n%QF16Base::k_step == 0) { ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_f16_f16_NxN(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ switch (nx) { ++ case 1: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ } ++ } else { ++ for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++ mul_mat_f16_f16_NxN(n, cx, bx, ix*k_nx, info); ++ } ++ int last_x = k_nx*(nrc_x/k_nx); ++ if (last_x == nrc_x) return; ++ int nx = nrc_x - last_x; ++ switch (nx) { ++ case 1: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 2: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 3: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ case 4: mul_mat_f16_f16_NxN(n, cx, bx, last_x, info); break; ++ } ++ } ++} ++ ++template ++IQK_NOINLINE void mul_mat_f16_f16_Nx1(int n, const char * cx, size_t bx, int ix0, const DataInfo& info) { ++ assert(n%QF16Base::k_step == 0); ++ int nb = n/QF16Base::k_step; ++ QF16<1> y(info); ++ QF16 x(cx + ix0*bx, bx); ++ QF16Base::Acc acc[4*nrc_x]; ++ auto yv = y.loadx(0, 0); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ for (int k = 0; k < 4; ++k) { ++ auto xv = x.load1(ix, k); ++ acc[4*ix+k] = QF16Base::acc_first(yv.val[k], xv); ++ } ++ } ++ for (int i = 1; i < nb/4; ++i) { ++ yv = y.loadx(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ for (int k = 0; k < 4; ++k) { ++ auto xv = x.load1(ix, 4*i+k); ++ acc[4*ix+k] = QF16Base::acc(acc[4*ix+k], yv.val[k], xv); ++ } ++ } ++ } ++ for (int i = 4*(nb/4); i < nb; ++i) { ++ auto yv1 = y.load1(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto xv1 = x.load1(ix, i); ++ acc[4*ix] = QF16Base::acc(acc[4*ix], yv1, xv1); ++ } ++ } ++ if constexpr (!is_multiple_of_k_step) { ++ int nb4 = n/4; ++ for (int i = (QF16Base::k_step/4)*nb; i < nb4; ++i) { ++ auto yv1 = y.load_tail(0, i); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto xv1 = x.load_tail(ix, i); ++ acc[4*ix] = QF16Base::acc(acc[4*ix], yv1, xv1); ++ } ++ } ++ } ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto v1 = vaddq_f16(acc[4*ix+0], acc[4*ix+1]); ++ auto v2 = vaddq_f16(acc[4*ix+2], acc[4*ix+3]); ++ info.store(ix0+ix, 0, QF16Base::hsum(vaddq_f16(v1, v2))); ++ } ++} ++ ++// At least on my M2-Max the version below, which does the multiplication row-by-row, is faster. ++// But let's keep this version commented out for now. ++//void mul_mat_f16_f16_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++// GGML_ASSERT(n%4 == 0); ++// constexpr int k_nx = 2; ++// const char * cx = (const char *)vx; ++// if (n%QF16Base::k_step == 0) { ++// for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++// mul_mat_f16_f16_Nx1(n, cx, bx, ix*k_nx, info); ++// } ++// int last_x = k_nx*(nrc_x/k_nx); ++// if (last_x == nrc_x) return; ++// int nx = nrc_x - last_x; ++// switch (nx) { ++// case 1: mul_mat_f16_f16_Nx1<1, true>(n, cx, bx, last_x, info); break; ++// //case 2: mul_mat_f16_f16_Nx1<2, true>(n, cx, bx, last_x, info); break; ++// //case 3: mul_mat_f16_f16_Nx1<3, true>(n, cx, bx, last_x, info); break; ++// } ++// } else { ++// for (int ix = 0; ix < nrc_x/k_nx; ++ix) { ++// mul_mat_f16_f16_Nx1(n, cx, bx, ix*k_nx, info); ++// } ++// int last_x = k_nx*(nrc_x/k_nx); ++// if (last_x == nrc_x) return; ++// int nx = nrc_x - last_x; ++// switch (nx) { ++// case 1: mul_mat_f16_f16_Nx1<1, false>(n, cx, bx, last_x, info); break; ++// //case 2: mul_mat_f16_f16_Nx1<2, false>(n, cx, bx, last_x, info); break; ++// //case 3: mul_mat_f16_f16_Nx1<3, false>(n, cx, bx, last_x, info); break; ++// } ++// } ++//} ++ ++void mul_mat_f16_f16_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%4 == 0); ++ const char * cx = (const char *)vx; ++ if (n%QF16Base::k_step == 0) { ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ mul_mat_f16_f16_Nx1<1, true>(n, cx, bx, ix, info); ++ } ++ } else { ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ mul_mat_f16_f16_Nx1<1, false>(n, cx, bx, ix, info); ++ } ++ } ++} ++ ++} ++ ++bool iqk_set_kernels_float(int ne00, int typeA, int typeB, std::array& kernels) { ++ ++ if (ne00%4 == 0) { ++ ++ if (typeA == GGML_TYPE_F16 && typeB == GGML_TYPE_F16) { ++ for (auto& f : kernels) f = nullptr; ++ kernels[0] = mul_mat_f16_f16_1; ++ kernels[1] = mul_mat_f16_f16_T<2>; ++ kernels[2] = mul_mat_f16_f16_T<3>; ++ kernels[3] = mul_mat_f16_f16_T<4>; ++ kernels[4] = mul_mat_f16_f16_T<5>; ++ return true; ++ } ++ else if (typeA == GGML_TYPE_BF16 && typeB == GGML_TYPE_F32) { ++ for (auto& f : kernels) f = nullptr; ++ kernels[0] = mul_mat_Qx_Qy_T, QBF16>; ++ kernels[1] = mul_mat_Qx_Qy_T, QBF16>; ++ kernels[2] = mul_mat_Qx_Qy_T, QBF16>; ++ kernels[3] = mul_mat_Qx_Qy_T, QBF16>; ++ kernels[4] = mul_mat_Qx_Qy_T, QBF16>; ++ return true; ++ } ++ ++ } ++ ++ return false; ++ ++} ++ ++namespace { ++template ++inline void mm_helper(int D, int nq, const char * cx, size_t bx, DataInfo& info, int k_step) { ++ constexpr int nrc_k = 6; ++ int krem = k_step - nrc_k*(k_step/nrc_k); ++ for (int iq = 0; iq < nq/nrc_q; ++iq) { ++ for (int ik = 0; ik < k_step/nrc_k; ++ik) { ++ mul_mat_f16_f16_NxN(D, cx, bx, ik*nrc_k, info); ++ } ++ if (krem > 0) { ++ switch (krem) { ++ case 1: mul_mat_f16_f16_NxN(D, cx, bx, k_step - krem, info); break; ++ case 2: mul_mat_f16_f16_NxN(D, cx, bx, k_step - krem, info); break; ++ case 3: mul_mat_f16_f16_NxN(D, cx, bx, k_step - krem, info); break; ++ case 4: mul_mat_f16_f16_NxN(D, cx, bx, k_step - krem, info); break; ++ default: mul_mat_f16_f16_NxN(D, cx, bx, k_step - krem, info); break; ++ } ++ } ++ info.cur_y += nrc_q; ++ } ++} ++} ++ ++void iqk_gemm_default_floats(int D, int nq, const char * cx, size_t bx, DataInfo& info, int k_step) { ++ constexpr int nrc_q = 4; ++ mm_helper(D, nq, cx, bx, info, k_step); ++ if (int qrem = nq - nrc_q*(nq/nrc_q); qrem > 0) { ++ switch (qrem) { ++ case 1: mm_helper<1>(D, nq, cx, bx, info, k_step); ++ case 2: mm_helper<2>(D, nq, cx, bx, info, k_step); ++ default: mm_helper<3>(D, nq, cx, bx, info, k_step); ++ } ++ } ++} ++ ++#endif ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_floats.h b/llama.cpp/ggml/src/iqk/iqk_gemm_floats.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_floats.h +@@ -0,0 +1,13 @@ ++#pragma once ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#include ++ ++bool iqk_set_kernels_float(int ne00, int typeA, int typeB, std::array& kernels); ++ ++void iqk_gemm_default_floats(int D, int nq, const char * vx, size_t bx, DataInfo& info, int k_step); ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.cpp b/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.cpp +@@ -0,0 +1,4829 @@ ++#include "iqk_gemm_kquants.h" ++#include ++ ++#ifdef IQK_IMPLEMENT ++ ++#include "ggml-impl.h" ++ ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++#include "ggml-quants.h" ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++// Handles q4_K and q5_K scales/mins ++struct Scales8K { ++ template ++ inline __m256i process_mins_and_scales(const uint8_t * data, float c, int i, const Q8& q8, __m256 * accd) { ++ make_q4_scales(data, utmp); ++ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0])); ++ const __m128i mins128 = _mm256_extracti128_si256(mins_and_scales, 1); ++ accum_mins(mins128, q8, i, c, accd); ++ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0); ++ return MM256_SET_M128I(sc128, sc128); ++ } ++#ifdef HAVE_FANCY_SIMD ++ template ++ inline __m512i process_mins_and_scales_64(const uint8_t * data, float c, int i, const Q8& q8, __m256 * accd) { ++ auto scales = process_mins_and_scales(data, c, i, q8, accd); ++ return _mm512_inserti32x8(_mm512_castsi256_si512(scales), scales, 1); ++ } ++#endif ++ template ++ inline void accum_mins(const __m128i& mins128, const Q8& q8, int i, float c, __m256 * accd) const { ++ base.accum_mins(mins128, q8, i, c, accd); ++ } ++#ifdef HAVE_FANCY_SIMD ++ const __m512i shuffles512[2] = { ++ _mm512_set_epi64(0x0706070607060706, 0x0302030203020302, 0x0706070607060706, 0x0302030203020302, ++ 0x0504050405040504, 0x0100010001000100, 0x0504050405040504, 0x0100010001000100), ++ _mm512_set_epi64(0x0f0e0f0e0f0e0f0e, 0x0b0a0b0a0b0a0b0a, 0x0f0e0f0e0f0e0f0e, 0x0b0a0b0a0b0a0b0a, ++ 0x0d0c0d0c0d0c0d0c, 0x0908090809080908, 0x0d0c0d0c0d0c0d0c, 0x0908090809080908) ++ }; ++#endif ++ Scales8KBase base; ++ ++ uint32_t utmp[4]; ++}; ++ ++template ++inline void process_mins_16(const __m256i& all_scales, const Q8& q8, int i, float d, __m256 * accm) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ const __m256i prod = _mm256_madd_epi16(all_scales, q8.load_bsums(iy, i)); ++ accm[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d * q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accm[iy]); ++ } ++} ++inline void prepare_scales_16(const __m256i& all_scales, __m256i * scales) { ++ const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0); ++ const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1); ++ scales[0] = MM256_SET_M128I(l_scales, l_scales); ++ scales[1] = MM256_SET_M128I(h_scales, h_scales); ++} ++ ++// Handles q3_K scales ++struct ScaleQ3 { ++ inline __m128i make_scales(const uint16_t * s8) const { ++ const uint16_t * scales16 = (const uint16_t *)s8; ++ uint32_t aux0 = scales16[0] | (scales16[1] << 16); ++ uint32_t aux1 = scales16[2] | (scales16[3] << 16); ++ uint32_t aux2 = scales16[4] | (scales16[5] << 16); ++ __m128i scales128 = _mm_set_epi32( ++ ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030), ++ ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030), ++ (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030), ++ (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030)); ++ return _mm_add_epi8(scales128, m32); ++ } ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct Scale16 { ++ inline void make_scales(const __m128i& scales8, __m512i * scales) const { ++ auto all_scales8 = MM256_SET_M128I(scales8, scales8); ++ auto scales1 = _mm256_shuffle_epi8(all_scales8, shuffle1); ++ auto scales2 = _mm256_shuffle_epi8(all_scales8, shuffle2); ++ scales[0] = _mm512_cvtepi8_epi16(scales1); ++ scales[1] = _mm512_cvtepi8_epi16(scales2); ++ } ++ template ++ inline void process_mins_and_scales(int i, float c, const __m128i& mins8, const __m128i& scales8, ++ const Q8& q8, __m256 * accm, __m512i * scales) const { ++ process_mins_16(_mm256_cvtepi8_epi16(mins8), q8, i, c, accm); ++ make_scales(scales8, scales); ++ } ++ const __m256i shuffle1 = _mm256_set_epi32(0x07070707, 0x03030303, 0x06060606, 0x02020202, ++ 0x05050505, 0x01010101, 0x04040404, 0x00000000); ++ const __m256i shuffle2 = _mm256_set_epi32(0x0f0f0f0f, 0x0b0b0b0b, 0x0e0e0e0e, 0x0a0a0a0a, ++ 0x0d0d0d0d, 0x09090909, 0x0c0c0c0c, 0x08080808); ++}; ++ ++template ++inline void process_mins_and_scales_16(const __m128i& scales128, const Q8& q8, int i, float d, ++ __m256 * accm, __m256i * scales) { ++ const __m256i all_scales = _mm256_cvtepi8_epi16(scales128); ++ process_mins_16(all_scales, q8, i, d, accm); ++ prepare_scales_16(all_scales, scales); ++} ++ ++inline __m256i get_scale_shuffle_8(int i) { ++ return _mm256_set1_epi16((2*i) | ((2*i+1) << 8)); ++} ++ ++inline void set_scales_8(const __m256i& all_scales, int j, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_8(4*j+3)); ++} ++ ++inline __m256i get_scale_shuffle_16(int i) { ++ static const uint8_t k_shuffle[128] = { ++ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, ++ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, ++ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11, ++ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15, ++ }; ++ return _mm256_loadu_si256((const __m256i*)k_shuffle + i); ++} ++ ++inline void set_scales_16(const __m256i& all_scales, __m256i * scales) { ++ scales[0] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(0)); ++ scales[1] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(1)); ++ scales[2] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(2)); ++ scales[3] = _mm256_shuffle_epi8(all_scales, get_scale_shuffle_16(3)); ++} ++ ++struct ScaleIQ4XS { ++ inline __m128i make_scales(const uint32_t scales_l, const uint16_t scales_h) { ++ uint32_t tmp32 = scales_h | (scales_h << 14); ++ const __m128i sh = _mm_slli_epi16(_mm_and_si128(_mm_srlv_epi32(_mm_set1_epi32(tmp32), hshift), hmask), 4); ++ const __m128i sl = _mm_and_si128(_mm_srlv_epi32(_mm_set1_epi32(scales_l), lshift), lmask); ++ return _mm_add_epi16(_mm_or_si128(sh, _mm_cvtepi8_epi16(_mm_shuffle_epi8(sl, lshuffle))), m32); ++ } ++ const __m128i hshift = _mm_set_epi32(12, 8, 4, 0); ++ const __m128i lshift = _mm_set_epi32(4, 0, 4, 0); ++ const __m128i hmask = _mm_set1_epi16(0x03); ++ const __m128i lmask = _mm_set1_epi8(0xf); ++ const __m128i lshuffle = _mm_set_epi32(0x07030602, 0x05010400, 0x07030602, 0x05010400); ++ const __m128i m32 = _mm_set1_epi16(-32); ++}; ++ ++#ifdef HAVE_FANCY_SIMD ++//====================================== Zen4 ================================================== ++ ++struct HighBit5 { ++ inline void apply(const uint8_t * h, Q4Bits& bits) { ++ auto hbits256 = _mm256_loadu_si256((const __m256i *)h); ++ auto hbits = _mm512_inserti32x8(_mm512_castsi256_si512(hbits256), _mm256_srli_epi16(hbits256, 1), 1); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_and_si512(_mm512_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_and_si512(hbits, mh)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh)); ++ } ++ const __m512i mh = _mm512_set1_epi8(0x10); ++}; ++ ++struct HighBit3 { ++ inline void apply(const uint8_t * h, Q2Bits& bits) { ++ auto hbits256 = _mm256_loadu_si256((const __m256i *)h); ++ auto hbits = _mm512_inserti32x8(_mm512_castsi256_si512(hbits256), _mm256_srli_epi16(hbits256, 1), 1); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh)); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_and_si512(hbits, mh)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_and_si512(_mm512_srli_epi16(hbits, 4), mh)); ++ } ++ const __m512i mh = _mm512_set1_epi8(0x04); ++}; ++ ++ ++template ++inline void compute_block(int iy, int i, float d, const Q8& q8, const __m512i * values, const __m512i * scales, __m512 * accd) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++} ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare(x[i].qs); ++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales); ++ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4); ++ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); ++ sc16.process_mins_and_scales(i, -GGML_FP16_TO_FP32(x[i].dmin), mins8, scales8, q8, accm, scales); ++ } ++ ++ Q2Bits bits; ++ Scale16 sc16; ++ const __m128i m4 = _mm_set1_epi8(0xf); ++ ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare(x[i].qs); ++ hbits.apply(x[i].hmask, bits); ++ auto scales128 = sc3.make_scales((const uint16_t *)x[i].scales); ++ sc16.process_mins_and_scales(i, -4.f*d, scales128, scales128, q8, accm, scales); ++ } ++ ++ Q2Bits bits; ++ HighBit3 hbits; ++ ScaleQ3 sc3; ++ Scale16 sc16; ++ const __m128i m4 = _mm_set1_epi8(0xf); ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare(x[i].qs); ++ auto all_scales = s8k.process_mins_and_scales_64(x[i].scales, -GGML_FP16_TO_FP32(x[i].dmin), i, q8, accd); ++ scales[0] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[0]); ++ scales[1] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[1]); ++ } ++ ++ Q4Bits bits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare(x[i].qs); ++ hbits.apply(x[i].qh, bits); ++ auto all_scales = s8k.process_mins_and_scales_64(x[i].scales, -GGML_FP16_TO_FP32(x[i].dmin), i, q8, accd); ++ scales[0] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[0]); ++ scales[1] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[1]); ++ } ++ ++ Q4Bits bits; ++ HighBit5 hbits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ bits.prepare64(x[i].ql); ++ add_high_bits(x[i].qh, bits); ++ auto scales128 = _mm_loadu_si128((const __m128i *)x[i].scales); ++ sc16.process_mins_and_scales(i, -32.f*d, scales128, scales128, q8, accm, scales); ++ } ++ ++ inline void add_high_bits(const uint8_t * qh, Q4Bits& bits) const { ++ auto hbits = _mm512_loadu_si512((const __m512i *)qh); ++ auto tmp1 = _mm512_and_si512(_mm512_slli_epi16(hbits, 4), mh); ++ auto tmp2 = _mm512_and_si512(_mm512_slli_epi16(hbits, 2), mh); ++ bits.values[0] = _mm512_or_si512(bits.values[0], _mm512_permutex2var_epi64(tmp1, bits.perm.permute1, tmp2)); ++ bits.values[2] = _mm512_or_si512(bits.values[2], _mm512_permutex2var_epi64(tmp1, bits.perm.permute2, tmp2)); ++ tmp1 = _mm512_and_si512(hbits, mh); ++ tmp2 = _mm512_and_si512(_mm512_srli_epi16(hbits, 2), mh); ++ bits.values[1] = _mm512_or_si512(bits.values[1], _mm512_permutex2var_epi64(tmp1, bits.perm.permute1, tmp2)); ++ bits.values[3] = _mm512_or_si512(bits.values[3], _mm512_permutex2var_epi64(tmp1, bits.perm.permute2, tmp2)); ++ } ++ ++ Q4Bits bits; ++ HighBit3 hbits; ++ Scale16 sc16; ++ ++ const __m512i mh = _mm512_set1_epi8(0x30); ++ ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ DequantizerIQ4XS(const void * vx, size_t bx) : BaseDequantizer(vx, bx), values(load_iq4nl_values_512()) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ prepare(x[i].qs); ++ auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); ++ s8k.accum_mins(scales128, q8, i, -128.f*d, accd); ++ auto scales256 = MM256_SET_M128I(scales128, scales128); ++ auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); ++ scales[0] = _mm512_shuffle_epi8(all_scales, shuffles[0]); ++ scales[1] = _mm512_shuffle_epi8(all_scales, shuffles[1]); ++ scales[2] = _mm512_shuffle_epi8(all_scales, shuffles[2]); ++ scales[3] = _mm512_shuffle_epi8(all_scales, shuffles[3]); ++ } ++ inline void prepare(const uint8_t * q4) { ++ bits.prepare64(q4); ++ // We now have in bits.valuse[0]: 0...15, 32...47, 64...79, 96...111 ++ // bits.valuse[1]: 16..31, 48...63, 80...95, 112..127 ++ // etc. ++ auto tmp = _mm512_permutex2var_epi64(bits.values[0], permute1, bits.values[1]); ++ bits.values[1] = _mm512_shuffle_epi8(values, _mm512_permutex2var_epi64(bits.values[0], permute2, bits.values[1])); ++ bits.values[0] = _mm512_shuffle_epi8(values, tmp); ++ tmp = _mm512_permutex2var_epi64(bits.values[2], permute1, bits.values[3]); ++ bits.values[3] = _mm512_shuffle_epi8(values, _mm512_permutex2var_epi64(bits.values[2], permute2, bits.values[3])); ++ bits.values[2] = _mm512_shuffle_epi8(values, tmp); ++ } ++ ++ Q4Bits bits; ++ Scales8KBase s8k; ++ ScaleIQ4XS siq4; ++ const __m512i values; ++ const __m512i permute1 = _mm512_set_epi64(11, 10, 3, 2, 9, 8, 1, 0); ++ const __m512i permute2 = _mm512_set_epi64(15, 14, 7, 6, 13, 12, 5, 4); ++ const __m512i shuffles[4] = { ++ _mm512_inserti32x8(_mm512_set1_epi16(0x0100), _mm256_set1_epi16(0x0302), 1), ++ _mm512_inserti32x8(_mm512_set1_epi16(0x0504), _mm256_set1_epi16(0x0706), 1), ++ _mm512_inserti32x8(_mm512_set1_epi16(0x0908), _mm256_set1_epi16(0x0b0a), 1), ++ _mm512_inserti32x8(_mm512_set1_epi16(0x0d0c), _mm256_set1_epi16(0x0f0e), 1), ++ }; ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_AVX512_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ constexpr int k_nx = 2; ++ ++ Q8<1> q8(info); ++ ++ Dequantizer deq1(vx, bx); ++ Dequantizer deq2(vx, bx); ++ ++ Dequantizer * deq[k_nx]; ++ deq[0] = &deq1; ++ deq[1] = &deq2; ++ ++ __m512i scales[2*k_nx]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ auto accd = _mm512_setzero_ps(); ++ auto accm = _mm256_setzero_ps(); ++ ++ for (int kx = 0; kx < k_nx; ++kx) deq[kx]->new_row(ix); ++ ++ for (int i = 0; i < nb/k_nx; ++i) { ++ ++ for (int kx = 0; kx < k_nx; ++kx) deq[kx]->new_block(k_nx*i+kx, q8, &accm, scales+2*kx); ++ ++ for (int kx = 0; kx < k_nx; ++kx) { ++ compute_block(0, k_nx*i+kx, deq[kx]->d, q8, deq[kx]->bits.values, scales+2*kx, &accd); ++ } ++ ++ } ++ if (2*(nb/2) < nb) { ++ int i0 = 2*(nb/2); ++ deq[0]->new_block(i0, q8, &accm, scales); ++ compute_block(0, i0, deq[0]->d, q8, deq[0]->bits.values, scales, &accd); ++ } ++ ++ auto sum256 = _mm256_add_ps(_mm512_castps512_ps256(accd), _mm512_extractf32x8_ps(accd, 1)); ++ info.store(ix, 0, hsum_float_8(_mm256_add_ps(accm, sum256))); ++ } ++} ++ ++template ++static void mul_mat_qX_K_q8_K_AVX512(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accm[nrc_y]; ++ __m512 accd[nrc_y]; ++ __m512i scales[2]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm512_setzero_ps(); ++ for (int iy = 0; iy < nrc_y; ++iy) accm[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.new_block(i, q8, accm, scales); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), deq.bits.values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(deq.d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum256 = _mm256_add_ps(_mm512_castps512_ps256(accd[iy]), _mm512_extractf32x8_ps(accd[iy], 1)); ++ info.store(ix, iy, hsum_float_8(_mm256_add_ps(accm[iy], sum256))); ++ } ++ ++ } ++} ++ ++template ++static void mul_mat_iqX_k_q8_K_AVX512(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accm[nrc_y]; ++ __m512 accd[nrc_y]; ++ __m512i scales[4]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm512_setzero_ps(); ++ for (int iy = 0; iy < nrc_y; ++iy) accm[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.new_block(i, q8, accm, scales); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m512i p1 = _mm512_maddubs_epi16(deq.bits.values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_maddubs_epi16(deq.bits.values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_maddubs_epi16(deq.bits.values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_maddubs_epi16(deq.bits.values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_dpwssd_epi32(_mm512_dpwssd_epi32(_mm512_dpwssd_epi32(_mm512_setzero_si512(), ++ p1, scales[0]), p2, scales[1]), p3, scales[2]), p4, scales[3]); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(deq.d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum256 = _mm256_add_ps(_mm512_castps512_ps256(accd[iy]), _mm512_extractf32x8_ps(accd[iy], 1)); ++ info.store(ix, iy, hsum_float_8(_mm256_add_ps(accm[iy], sum256))); ++ } ++ ++ } ++} ++ ++#else ++//====================================== AVX2 ================================================== ++ ++struct HighBit5 { ++ inline void load(const uint8_t * h) { hbits = _mm256_loadu_si256((const __m256i *)h); } ++ inline void apply(Q4Bits& bits, bool do_shift) { ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 3), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ if (do_shift) { ++ hbits = _mm256_srli_epi16(hbits, 4); ++ } ++ } ++ const __m256i mh = _mm256_set1_epi8(0x10); ++ __m256i hbits; ++}; ++ ++struct HighBit3 { ++ inline void load(const uint8_t * h) { hbits = _mm256_loadu_si256((const __m256i *)h); } ++ inline void apply(Q2Bits& bits, bool do_shift) { ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh)); ++ if (do_shift) { ++ hbits = _mm256_srli_epi16(hbits, 4); ++ } ++ } ++ const __m256i mh = _mm256_set1_epi8(0x04); ++ __m256i hbits; ++}; ++ ++template ++inline void compute_block(int iy, int i, float d, const Q8& q8, const __m512i * values, const __m512i * scales, __m512 * accd) { ++ const __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[0], q8.load_quants64(iy, i, 0)); ++ const __m512i p2 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[1], q8.load_quants64(iy, i, 1)); ++ const __m512i p3 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[2], q8.load_quants64(iy, i, 2)); ++ const __m512i p4 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), values[3], q8.load_quants64(iy, i, 3)); ++ auto sumi = _mm512_dpwssd_epi32(_mm512_setzero_si512(), scales[0], _mm512_packs_epi32(p1, p2)); ++ sumi = _mm512_dpwssd_epi32(sumi, scales[1], _mm512_packs_epi32(p3, p4)); ++ accd[iy] = _mm512_fmadd_ps(_mm512_set1_ps(d*q8.scale(iy, i)), _mm512_cvtepi32_ps(sumi), accd[iy]); ++} ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales); ++ const __m128i scales8 = _mm_and_si128(mins_and_scales, m4); ++ const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); ++ process_mins_16(_mm256_cvtepi8_epi16(mins8), q8, i, -GGML_FP16_TO_FP32(x[i].dmin), accm); ++ prepare_scales_16(_mm256_cvtepi8_epi16(scales8), scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ } ++ ++ Q2Bits bits; ++ ++ const __m128i m4 = _mm_set1_epi8(0xf); ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ hbits.load(x[i].hmask); ++ process_mins_and_scales_16(sc3.make_scales((const uint16_t *)x[i].scales), q8, i, -4.f*d, accm, scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ hbits.apply(bits, j == 0); ++ } ++ ++ Q2Bits bits; ++ HighBit3 hbits; ++ ScaleQ3 sc3; ++ ++ const __m128i m32 = _mm_set1_epi8(-32); ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline __m256i new_block(int i, const Q8& q8, __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ return s8k.process_mins_and_scales(x[i].scales, -GGML_FP16_TO_FP32(x[i].dmin), i, q8, accd); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ } ++ ++ Q4Bits bits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline __m256i new_block(int i, const Q8& q8, __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ hbits.load(x[i].qh); ++ return s8k.process_mins_and_scales(x[i].scales, -GGML_FP16_TO_FP32(x[i].dmin), i, q8, accd); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ hbits.apply(bits, j == 0); ++ } ++ ++ Q4Bits bits; ++ HighBit5 hbits; ++ Scales8K s8k; ++}; ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ template ++ inline void new_block(int i, const Q8& q8, __m256 * accm, __m256i * scales) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ process_mins_and_scales_16(_mm_loadu_si128((const __m128i *)x[i].scales), q8, i, -32.f*d, accm, scales); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare64(x[i].ql, j); ++ auto hbits = _mm256_loadu_si256((const __m256i *)x[i].qh + j); ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_srli_epi16(hbits, 2), mh)); ++ } ++ ++ Q4Bits bits; ++ const __m256i mh = _mm256_set1_epi8(0x30); ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ DequantizerIQ4XS(const void * vx, size_t bx) : BaseDequantizer(vx, bx), values(load_iq4nl_values_256()) {} ++ template ++ inline __m256i new_block(int i, const Q8& q8, __m256 * accd) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); ++ s8k.accum_mins(scales128, q8, i, -128.f*d, accd); ++ return MM256_SET_M128I(scales128, scales128); ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare16(x[i].qs, j); ++ bits.values[0] = _mm256_shuffle_epi8(values, bits.values[0]); ++ bits.values[1] = _mm256_shuffle_epi8(values, bits.values[1]); ++ bits.values[2] = _mm256_shuffle_epi8(values, bits.values[2]); ++ bits.values[3] = _mm256_shuffle_epi8(values, bits.values[3]); ++ } ++ ++ Q4Bits bits; ++ Scales8K s8k; ++ ScaleIQ4XS siq4; ++ const __m256i values; ++}; ++ ++template ++static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accd[nrc_y]; ++ __m256i scales[4]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ auto all_scales = deq.new_block(i, q8, accd); ++ ++ __m256i sumi[nrc_y]; ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ ++ deq.prepare(i, j); ++ ++ set_scales_8(all_scales, j, scales); ++ ++ multiply_add(deq.bits, scales, j, i, q8, sumi); ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const __m256 vd = _mm256_set1_ps(deq.d*q8.scale(iy, i)); ++ accd[iy] = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi[iy]), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++} ++ ++template ++static void mul_mat_qY_K_q8_K_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%QK_K == 0); ++ const int nb = n/QK_K; ++ ++ Q8 q8(info); ++ ++ __m256i all_scales[2]; ++ __m256i scales[4]; ++ __m256 accd[nrc_y]; ++ ++ Dequantizer deq(vx, bx); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.new_block(i, q8, accd, all_scales); ++ ++ __m256i sumi[nrc_y]; ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ deq.prepare(i, j); ++ set_scales_16(all_scales[j], scales); ++ multiply_add(deq.bits, scales, j, i, q8, sumi); ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ accd[iy] = _mm256_fmadd_ps(_mm256_set1_ps(deq.d*q8.scale(iy, i)), _mm256_cvtepi32_ps(sumi[iy]), accd[iy]); ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++ ++} ++ ++#endif ++ ++// inline __m256i process_mins_and_scales(const uint8_t * data, float c, int i, const Q8& q8, __m256 * accd) { ++// make_q4_scales(data, utmp); ++// const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0])); ++// const __m128i mins128 = _mm256_extracti128_si256(mins_and_scales, 1); ++// accum_mins(mins128, q8, i, c, accd); ++// const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0); ++// return MM256_SET_M128I(sc128, sc128); ++// } ++// ++// inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { ++// d = GGML_FP16_TO_FP32(x[i].d); ++// bits.prepare(x[i].qs); ++// auto all_scales = s8k.process_mins_and_scales_64(x[i].scales, -GGML_FP16_TO_FP32(x[i].dmin), i, q8, accd); ++// scales[0] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[0]); ++// scales[1] = _mm512_shuffle_epi8(all_scales, s8k.shuffles512[1]); ++// } ++ ++ ++struct Q4Bits_AVX2 { ++ inline void prepare(const uint8_t * q4, int j) { ++ auto q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+0); ++ values[0] = _mm256_and_si256(q4bits, ml); ++ values[1] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ q4bits = _mm256_loadu_si256((const __m256i*)q4 + 2*j+1); ++ values[2] = _mm256_and_si256(q4bits, ml); ++ values[3] = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), ml); ++ } ++ __m256i values[4]; ++ const __m256i ml = _mm256_set1_epi8(0xf); ++}; ++ ++struct DequantizerQ4K_AVX2 final : public BaseDequantizer { ++ DequantizerQ4K_AVX2(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ } ++ Q4Bits_AVX2 bits; ++}; ++ ++struct DequantizerQ5K_AVX2 final : public BaseDequantizer { ++ DequantizerQ5K_AVX2(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs, j); ++ hbits = j == 0 ? _mm256_loadu_si256((const __m256i *)x[i].qh) : _mm256_srli_epi16(hbits, 4); ++ apply_hbits(); ++ } ++ inline void apply_hbits() { ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 3), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ } ++ ++ const __m256i mh = _mm256_set1_epi8(0x10); ++ Q4Bits_AVX2 bits; ++ __m256i hbits; ++}; ++ ++template ++static void mul_mat_qX_K_q8_2_X4_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ uint32_t utmp[4]; ++ __m256 accd[nrc_y]; ++ __m256 scales[2]; ++ float d8[8*nrc_y]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.d = GGML_FP16_TO_FP32(deq.x[i].d); ++ auto vm = _mm256_cvtph_ps(_mm_set1_epi16(deq.x[i].dmin)); ++ make_q4_scales(deq.x[i].scales, utmp); ++ auto mins = _mm256_mul_ps(vm, _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(utmp + 2))))); ++ mins = _mm256_mul_ps(_mm256_set1_ps(-1.f), mins); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4_1 = _mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+0].d))); ++ auto d4_2 = _mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+1].d))); ++ auto dy = _mm256_castsi256_ps(_mm256_slli_epi32(MM256_SET_M128I(d4_2, d4_1), 16)); ++ _mm256_storeu_ps(d8 + 8*iy, dy); ++ auto m4_1 = _mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+0].d+4))); ++ auto m4_2 = _mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+1].d+4))); ++ auto myi = MM256_SET_M128I(m4_2, m4_1); ++ auto my = _mm256_mul_ps(dy, _mm256_cvtepi32_ps(myi)); ++ accd[iy] = _mm256_fmadd_ps(my, mins, accd[iy]); ++ } ++ ++ auto all_scales = _mm256_mul_ps(_mm256_set1_ps(deq.d), _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)utmp)))); ++ scales[0] = _mm256_set_m128(_mm256_castps256_ps128(all_scales), _mm256_castps256_ps128(all_scales)); ++ auto scales_h = _mm256_extractf128_ps(all_scales, 1); ++ scales[1] = _mm256_set_m128(scales_h, scales_h); ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ ++ deq.prepare(i, j); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ const block_q8_2_x4& y = q8.y[iy][2*i+j]; ++#ifdef HAVE_FANCY_SIMD ++ auto sumi1 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), deq.bits.values[0], _mm256_loadu_si256((const __m256i*)y.qs+0)); ++ auto sumi2 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), deq.bits.values[1], _mm256_loadu_si256((const __m256i*)y.qs+1)); ++ auto sumi3 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), deq.bits.values[2], _mm256_loadu_si256((const __m256i*)y.qs+2)); ++ auto sumi4 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), deq.bits.values[3], _mm256_loadu_si256((const __m256i*)y.qs+3)); ++ sumi1 = _mm256_add_epi32(_mm256_unpacklo_epi32(sumi1, sumi2), _mm256_unpackhi_epi32(sumi1, sumi2)); ++ sumi3 = _mm256_add_epi32(_mm256_unpacklo_epi32(sumi3, sumi4), _mm256_unpackhi_epi32(sumi3, sumi4)); ++ sumi1 = _mm256_add_epi32(_mm256_unpacklo_epi64(sumi1, sumi3), _mm256_unpackhi_epi64(sumi1, sumi3)); ++#else ++ auto sumi1 = _mm256_maddubs_epi16(deq.bits.values[0], _mm256_loadu_si256((const __m256i*)y.qs+0)); ++ auto sumi2 = _mm256_maddubs_epi16(deq.bits.values[1], _mm256_loadu_si256((const __m256i*)y.qs+1)); ++ auto sumi3 = _mm256_maddubs_epi16(deq.bits.values[2], _mm256_loadu_si256((const __m256i*)y.qs+2)); ++ auto sumi4 = _mm256_maddubs_epi16(deq.bits.values[3], _mm256_loadu_si256((const __m256i*)y.qs+3)); ++ sumi1 = _mm256_add_epi16(_mm256_unpacklo_epi32(sumi1, sumi2), _mm256_unpackhi_epi32(sumi1, sumi2)); ++ sumi3 = _mm256_add_epi16(_mm256_unpacklo_epi32(sumi3, sumi4), _mm256_unpackhi_epi32(sumi3, sumi4)); ++ sumi1 = _mm256_add_epi16(_mm256_unpacklo_epi64(sumi1, sumi3), _mm256_unpackhi_epi64(sumi1, sumi3)); ++ sumi1 = _mm256_madd_epi16(_mm256_set1_epi16(1), sumi1); ++#endif ++ auto dy4 = _mm_loadu_ps(d8 + 8*iy + 4*j); ++ auto d4d8 = _mm256_mul_ps(scales[j], _mm256_set_m128(dy4, dy4)); ++ accd[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi1), accd[iy]); ++ } ++ ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++} ++ ++struct DequantizerQ6K_AVX2 final : public BaseDequantizer { ++ DequantizerQ6K_AVX2(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ inline void prepare(int i, int j) { ++ auto lbits1 = _mm256_loadu_si256((const __m256i *)x[i].ql + 2*j+0); ++ auto lbits2 = _mm256_loadu_si256((const __m256i *)x[i].ql + 2*j+1); ++ auto hbits = _mm256_loadu_si256((const __m256i *)x[i].qh + j); ++ bits.values[0] = _mm256_or_si256(_mm256_and_si256(lbits1, bits.ml), _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ bits.values[1] = _mm256_or_si256(_mm256_and_si256(lbits2, bits.ml), _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits1, 4), bits.ml), _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits2, 4), bits.ml), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), mh)); ++ } ++ inline void prepare_signed(int i, int j, __m256i * us) { ++ prepare(i, j); ++ for (int k = 0; k < 4; ++k) { ++ bits.values[k] = _mm256_add_epi8(bits.values[k], _mm256_set1_epi8(-32)); ++ us[k] = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ } ++ } ++ inline __m256i make_scales(int i) const { ++ return _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i *)x[i].scales)); ++ } ++ ++ const __m256i mh = _mm256_set1_epi8(0x30); ++ Q4Bits_AVX2 bits; ++}; ++ ++struct SimpleBits { ++ __m256i values[4]; ++}; ++ ++struct DequantizerQ3K_AVX2 final : public BaseDequantizer { ++ DequantizerQ3K_AVX2(const void * vx, size_t bx) : BaseDequantizer(vx, bx) {} ++ ++ inline void prepare(int i, int j) { ++ hbits = j == 0 ? _mm256_loadu_si256((const __m256i *)x[i].hmask) : _mm256_srli_epi16(hbits, 4); ++ auto q2bits = _mm256_loadu_si256((const __m256i *)x[i].qs + j); ++ bits.values[0] = _mm256_and_si256(q2bits, ml); ++ bits.values[1] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), ml); ++ bits.values[2] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), ml); ++ bits.values[3] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), ml); ++ bits.values[0] = _mm256_or_si256(bits.values[0], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ bits.values[1] = _mm256_or_si256(bits.values[1], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ bits.values[2] = _mm256_or_si256(bits.values[2], _mm256_and_si256(hbits, mh)); ++ bits.values[3] = _mm256_or_si256(bits.values[3], _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh)); ++ //bits.values[0] = _mm256_sub_epi8(bits.values[0], _mm256_xor_si256(mh, _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh))); ++ //bits.values[1] = _mm256_sub_epi8(bits.values[1], _mm256_xor_si256(mh, _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh))); ++ //bits.values[2] = _mm256_sub_epi8(bits.values[2], _mm256_xor_si256(mh, _mm256_and_si256(hbits, mh))); ++ //bits.values[3] = _mm256_sub_epi8(bits.values[3], _mm256_xor_si256(mh, _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh))); ++ } ++ inline void prepare_signed(int i, int j, __m256i * us) { ++ prepare(i, j); ++ for (int k = 0; k < 4; ++k) { ++ bits.values[k] = _mm256_sub_epi8(bits.values[k], mh); ++ us[k] = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ } ++ //for (int k = 0; k < 4; ++k) { ++ // us[k] = _mm256_sign_epi8(bits.values[k], bits.values[k]); ++ //} ++ } ++ inline __m256i make_scales(int i) const { ++ return _mm256_cvtepi8_epi16(sc3.make_scales((const uint16_t *)x[i].scales)); ++ } ++ ++ ScaleQ3 sc3; ++ ++ __m256i hbits; ++ SimpleBits bits; ++ const __m256i ml = _mm256_set1_epi8(3); ++ const __m256i mh = _mm256_set1_epi8(4); ++}; ++ ++template ++static void mul_mat_qY_K_q8_2_X4_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx); ++ ++ __m256 accd[nrc_y]; ++ __m256 scales[2]; ++ float d8[8*nrc_y]; ++ __m256i us[4]; ++ ++ uint8_t k_shuff[32] = {0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; ++ auto shuff = _mm256_loadu_si256((const __m256i *)k_shuff); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ for (int iy = 0; iy < nrc_y; ++iy) accd[iy] = _mm256_setzero_ps(); ++ ++ deq.new_row(ix); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ deq.d = GGML_FP16_TO_FP32(deq.x[i].d); ++ auto vd = _mm256_set1_ps(deq.d); ++ auto sc16 = _mm256_shuffle_epi8(deq.make_scales(i), shuff); ++ scales[0] = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(sc16)))); ++ scales[1] = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(sc16, 1)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4_1 = _mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+0].d))); ++ auto d4_2 = _mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][2*i+1].d))); ++ auto dy = _mm256_castsi256_ps(_mm256_slli_epi32(MM256_SET_M128I(d4_2, d4_1), 16)); ++ if constexpr (nrc_y == 1) { ++ auto dyh = _mm256_extractf128_ps(dy, 1); ++ scales[0] = _mm256_mul_ps(scales[0], _mm256_set_m128(_mm256_castps256_ps128(dy), _mm256_castps256_ps128(dy))); ++ scales[1] = _mm256_mul_ps(scales[1], _mm256_set_m128(dyh, dyh)); ++ } else { ++ _mm256_storeu_ps(d8 + 8*iy, dy); ++ } ++ } ++ ++ for (int j = 0; j < QK_K/128; ++j) { ++ ++ deq.prepare_signed(i, j, us); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qs = q8.y[iy][2*i+j].qs; ++#ifdef HAVE_FANCY_SIMD ++ // 0...31 ++ auto sumi1 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), us[0], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+0), deq.bits.values[0])); ++ // 32...63 ++ auto sumi2 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), us[1], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+1), deq.bits.values[1])); ++ // 64...95 ++ auto sumi3 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), us[2], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+2), deq.bits.values[2])); ++ // 96...128 ++ auto sumi4 = _mm256_dpbusd_epi32(_mm256_setzero_si256(), us[3], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+3), deq.bits.values[3])); ++ // 0...3, 32...35, 4....7, 36...39, 16...19, 48...51, 20...23, 52...56 + ++ // 8..11, 40...43, 12...15, 44...47, 24...27, 56...59, 28...31, 60...63 ++ // b0 b2 b0 b2 b1 b3 b1 b3 ++ sumi1 = _mm256_add_epi32(_mm256_unpacklo_epi32(sumi1, sumi2), _mm256_unpackhi_epi32(sumi1, sumi2)); ++ // same as above + 64, so ++ // b4 b6, b4 b6 b5 b7 b5 b7 ++ sumi3 = _mm256_add_epi32(_mm256_unpacklo_epi32(sumi3, sumi4), _mm256_unpackhi_epi32(sumi3, sumi4)); ++ // b0 b2 b4 b6 b1 b3 b5 b7 + ++ // b0 b2 b4 b6 b1 b3 b5 b7 ++ sumi1 = _mm256_add_epi32(_mm256_unpacklo_epi64(sumi1, sumi3), _mm256_unpackhi_epi64(sumi1, sumi3)); ++#else ++ auto sumi1 = _mm256_maddubs_epi16(us[0], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+0), deq.bits.values[0])); ++ auto sumi2 = _mm256_maddubs_epi16(us[1], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+1), deq.bits.values[1])); ++ auto sumi3 = _mm256_maddubs_epi16(us[2], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+2), deq.bits.values[2])); ++ auto sumi4 = _mm256_maddubs_epi16(us[3], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)qs+3), deq.bits.values[3])); ++ sumi1 = _mm256_add_epi16(_mm256_unpacklo_epi32(sumi1, sumi2), _mm256_unpackhi_epi32(sumi1, sumi2)); ++ sumi3 = _mm256_add_epi16(_mm256_unpacklo_epi32(sumi3, sumi4), _mm256_unpackhi_epi32(sumi3, sumi4)); ++ sumi1 = _mm256_add_epi16(_mm256_unpacklo_epi64(sumi1, sumi3), _mm256_unpackhi_epi64(sumi1, sumi3)); ++ sumi1 = _mm256_madd_epi16(_mm256_set1_epi16(1), sumi1); ++#endif ++ if constexpr (nrc_y > 1) { ++ auto dy4 = _mm_loadu_ps(d8 + 8*iy + 4*j); ++ auto d4d8 = _mm256_mul_ps(scales[j], _mm256_set_m128(dy4, dy4)); ++ accd[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi1), accd[iy]); ++ } else { ++ accd[iy] = _mm256_fmadd_ps(scales[j], _mm256_cvtepi32_ps(sumi1), accd[iy]); ++ } ++ } ++ ++ } ++ ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, hsum_float_8(accd[iy])); ++ } ++ ++ } ++} ++ ++template ++static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ auto m30 = _mm256_set1_epi8(0x30); ++ auto m32 = _mm256_set1_epi8(32); ++#ifndef HAVE_FANCY_SIMD ++ auto s_shuffle = _mm256_set_epi64x(0x0f0e0f0e0d0c0d0c, 0x0b0a0b0a09080908, 0x0706070605040504, 0x0302030201000100); ++ auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); ++ auto values = MM256_SET_M128I(values128, values128); ++#else ++ auto values = load_iq4nl_values_256(); ++#endif ++ int nbl = n / QK_K; ++ using helper_t = union { __m256i vec[2]; uint64_t val[8]; }; ++ helper_t h; ++ __m256 acc[nrc_y] = {}; ++ __m256i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_xs_r8 * iq4 = (const block_iq4_xs_r8 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto d4 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ibl].d)); ++ auto slbits = _mm256_loadu_si256((const __m256i *)iq4[ibl].scales_l); ++ auto sl1 = _mm256_and_si256(slbits, m4); ++ auto sl2 = _mm256_and_si256(_mm256_srli_epi16(slbits, 4), m4); ++ auto shbits = _mm_loadu_si128((const __m128i*)iq4[ibl].scales_h); ++ auto sh = MM256_SET_M128I(_mm_srli_epi16(shbits, 2), shbits); ++ h.vec[0] = _mm256_sub_epi8(_mm256_or_si256(sl1, _mm256_and_si256(_mm256_slli_epi16(sh, 4), m30)), m32); ++ h.vec[1] = _mm256_sub_epi8(_mm256_or_si256(sl2, _mm256_and_si256(sh, m30)), m32); ++ __m256i isum[nrc_y] = {}; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++#ifdef HAVE_FANCY_SIMD ++ auto iscales = _mm256_cvtepi8_epi32(_mm_set1_epi64x(h.val[ib])); ++ auto scales = _mm256_mul_ps(d4, _mm256_cvtepi32_ps(iscales)); ++ auto scales_m = _mm256_mul_ps(scales, _mm256_set1_ps(-128.f)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ float m8 = ((const float *)q8.y[iy][ibl].bsums)[ib]; ++ acc[iy] = _mm256_fmadd_ps(scales_m, _mm256_set1_ps(m8), acc[iy]); ++ } ++#else ++ auto iscales = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm_set1_epi64x(h.val[ib])), s_shuffle); ++#endif ++ auto bits1 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+4*ib+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+4*ib+1); ++ qx[0] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits1)); ++ qx[1] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits1, 4))); ++ qx[2] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits2)); ++ qx[3] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits2, 4))); ++#ifndef HAVE_FANCY_SIMD ++ auto s1 = _mm256_sign_epi8(qx[0], qx[0]); ++ auto s2 = _mm256_sign_epi8(qx[1], qx[1]); ++ auto s3 = _mm256_sign_epi8(qx[2], qx[2]); ++ auto s4 = _mm256_sign_epi8(qx[3], qx[3]); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+0); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi)); ++#else ++ auto sumi1 = _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ auto sumi2 = _mm256_maddubs_epi16(s2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ auto sumi3 = _mm256_maddubs_epi16(s3, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ auto sumi4 = _mm256_maddubs_epi16(s4, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++ auto sumi = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(iscales, sumi1), _mm256_madd_epi16(iscales, sumi2)), ++ _mm256_add_epi32(_mm256_madd_epi16(iscales, sumi3), _mm256_madd_epi16(iscales, sumi4))); ++ isum[iy] = _mm256_add_epi32(isum[iy], sumi); ++#endif ++ } ++ bits1 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+4*ib+2); ++ bits2 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+4*ib+3); ++ qx[0] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits1)); ++ qx[1] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits1, 4))); ++ qx[2] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits2)); ++ qx[3] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits2, 4))); ++#ifndef HAVE_FANCY_SIMD ++ s1 = _mm256_sign_epi8(qx[0], qx[0]); ++ s2 = _mm256_sign_epi8(qx[1], qx[1]); ++ s3 = _mm256_sign_epi8(qx[2], qx[2]); ++ s4 = _mm256_sign_epi8(qx[3], qx[3]); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+1); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi)); ++#else ++ auto sumi1 = _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ auto sumi2 = _mm256_maddubs_epi16(s2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ auto sumi3 = _mm256_maddubs_epi16(s3, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ auto sumi4 = _mm256_maddubs_epi16(s4, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++ auto sumi = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(iscales, sumi1), _mm256_madd_epi16(iscales, sumi2)), ++ _mm256_add_epi32(_mm256_madd_epi16(iscales, sumi3), _mm256_madd_epi16(iscales, sumi4))); ++ isum[iy] = _mm256_add_epi32(isum[iy], sumi); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_iq4_xs_r8_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ mul_mat_iq4_xs_r8_q8_k_avx2(n, vx, bx, info, nrc_x); ++ return; ++ if constexpr (nrc_y == 1){ ++ mul_mat_iq4_xs_r8_q8_k_avx2<1>(n, vx, bx, info, nrc_x); ++ } else { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm512_set1_epi8(0xf); ++ auto values = load_iq4nl_values_512(); ++ int nbl = n / QK_K; ++ using helper_t = union { __m512i vec; uint32_t val[16]; }; ++ helper_t h; ++ __m512 acc[nrc_y] = {}; ++ __m512i isum[nrc_y] = {}; ++ __m512i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_xs_r8 * iq4l = (const block_iq4_xs_r8 *)((const char *)vx + (ix+0)*bx); ++ const block_iq4_xs_r8 * iq4h = (const block_iq4_xs_r8 *)((const char *)vx + (ix+4)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dl = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq4l[ibl].d)); ++ auto dh = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq4h[ibl].d)); ++ auto d4 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_set_m128(dl, dl)), _mm256_set_m128(dh, dh), 1); ++ auto d4x64 = _mm512_mul_ps(d4, _mm512_set1_ps(-64.f)); ++ auto slbits_l = _mm_loadu_si128((const __m128i *)iq4l[ibl].scales_l); ++ auto shbits_l = _mm_loadu_si128((const __m128i *)iq4h[ibl].scales_l); ++ auto sl_l = MM256_SET_M128I(_mm_srli_epi16(slbits_l, 4), slbits_l); ++ auto sh_l = MM256_SET_M128I(_mm_srli_epi16(shbits_l, 4), shbits_l); ++ auto slb = _mm512_and_si512(_mm512_inserti32x8(_mm512_castsi256_si512(sl_l), sh_l, 1), m4); ++ auto aux64 = (const uint64_t *)iq4l[ibl].scales_h; ++ auto slbits_h = _mm_set_epi64x(aux64[0] >> 2, aux64[0]); ++ aux64 = (const uint64_t *)iq4h[ibl].scales_h; ++ auto shbits_h = _mm_set_epi64x(aux64[0] >> 2, aux64[0]); ++ auto sl_h = MM256_SET_M128I(slbits_h, _mm_slli_epi16(slbits_h, 4)); ++ auto sh_h = MM256_SET_M128I(shbits_h, _mm_slli_epi16(shbits_h, 4)); ++ auto shb = _mm512_and_si512(_mm512_inserti32x8(_mm512_castsi256_si512(sl_h), sh_h, 1), _mm512_set1_epi8(0x30)); ++ h.vec = _mm512_sub_epi8(_mm512_or_si512(slb, shb), _mm512_set1_epi8(32)); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ auto iscales = _mm512_cvtepi8_epi32(_mm_blend_epi32(_mm_set1_epi32(h.val[ib+0]), _mm_set1_epi32(h.val[ib+8]), 0x0c)); ++ auto scales = _mm512_cvtepi32_ps(iscales); ++ auto scales_m = _mm512_mul_ps(scales, d4x64); ++ auto bits1 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq4l[ibl].qs+2*ib+0)), ++ _mm256_loadu_si256((const __m256i *)iq4h[ibl].qs+2*ib+0), 1); ++ auto bits2 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq4l[ibl].qs+2*ib+1)), ++ _mm256_loadu_si256((const __m256i *)iq4h[ibl].qs+2*ib+1), 1); ++ qx[0] = _mm512_shuffle_epi8(values, _mm512_and_si512(bits1, m4)); ++ qx[1] = _mm512_shuffle_epi8(values, _mm512_and_si512(bits2, m4)); ++ qx[2] = _mm512_shuffle_epi8(values, _mm512_and_si512(_mm512_srli_epi16(bits1, 4), m4)); ++ qx[3] = _mm512_shuffle_epi8(values, _mm512_and_si512(_mm512_srli_epi16(bits2, 4), m4)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y8 = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y8), y8, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ isum[iy] = _mm512_add_epi32(isum[iy], _mm512_mullo_epi32(iscales, sumi)); ++ float m8 = ((const float *)q8.y[iy][ibl].bsums)[ib]; ++ acc[iy] = _mm512_fmadd_ps(scales_m, _mm512_set1_ps(m8), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = _mm512_fmadd_ps(_mm512_mul_ps(d4, _mm512_set1_ps(q8.scale(iy, ibl))), _mm512_cvtepi32_ps(isum[iy]), acc[iy]); ++ isum[iy] = _mm512_setzero_si512(); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum1 = _mm_add_ps(_mm512_extractf32x4_ps(acc[iy], 0), _mm512_extractf32x4_ps(acc[iy], 1)); ++ auto sum2 = _mm_add_ps(_mm512_extractf32x4_ps(acc[iy], 2), _mm512_extractf32x4_ps(acc[iy], 3)); ++ info.store(ix+0, iy, sum1); ++ info.store(ix+4, iy, sum2); ++ acc[iy] = _mm512_setzero_ps(); ++ } ++ } ++ } ++} ++#else ++template ++static void mul_mat_iq4_xs_r8_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ mul_mat_iq4_xs_r8_q8_k_avx2(n, vx, bx, info, nrc_x); ++} ++#endif ++ ++template ++static void mul_mat_q2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mxf = _mm256_set1_epi8(0xf); ++ auto m03 = _mm256_set1_epi8(0x03); ++ static const uint8_t k_shuff[32] = {0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15}; ++ auto shuff = _mm256_loadu_si256((const __m256i *)k_shuff); ++#ifdef HAVE_FANCY_SIMD ++ __m256i isum[nrc_y] = {}; ++#else ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nbl = n / QK_K; ++ __m256 acc[nrc_y] = {}; ++ __m256i qx[4]; ++ int8_t scales[64]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q2_k_r4 * iq2 = (const block_q2_k_r4 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dm = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq2[ibl].d)); ++ auto d4 = _mm256_set_m128(_mm256_castps256_ps128(dm), _mm256_castps256_ps128(dm)); ++ auto m4 = _mm256_set_m128(_mm256_extractf128_ps(dm, 1), _mm256_extractf128_ps(dm, 1)); ++ m4 = _mm256_mul_ps(m4, _mm256_set1_ps(-1.f)); ++ auto all_scales1 = _mm256_loadu_si256((const __m256i *)iq2[ibl].scales+0); ++ auto all_scales2 = _mm256_loadu_si256((const __m256i *)iq2[ibl].scales+1); ++ auto scales1 = _mm256_and_si256(_mm256_srli_epi16(all_scales1, 4), mxf); ++ auto scales2 = _mm256_and_si256(_mm256_srli_epi16(all_scales2, 4), mxf); ++ { ++ auto t1 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales1, 0)), shuff); // blocks 0, 1, 2, 3 for each row ++ auto t2 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales1, 1)), shuff); // blocks 4, 5, 6, 7 for each row ++ auto t3 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales2, 0)), shuff); // blocks 8, 9, 10, 11 for each row ++ auto t4 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales2, 1)), shuff); // blocks 12, 13, 14, 15 for each row ++ auto s1 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 0), _mm256_extracti128_si256(t1, 0)); // blocks 0, 1, 8, 9 ++ auto s2 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 1), _mm256_extracti128_si256(t1, 1)); // blocks 2, 3, 10, 11 ++ auto s3 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 0), _mm256_extracti128_si256(t2, 0)); // blocks 4, 5, 12, 13 ++ auto s4 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 1), _mm256_extracti128_si256(t2, 1)); // blocks 6, 7, 14, 15 ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums(iy, ibl); ++ auto sumi = _mm256_setzero_si256(); ++#ifdef HAVE_FANCY_SIMD ++ sumi = _mm256_dpwssd_epi32(sumi, s1, _mm256_shuffle_epi32(bsums, 0x00)); ++ sumi = _mm256_dpwssd_epi32(sumi, s2, _mm256_shuffle_epi32(bsums, 0x55)); ++ sumi = _mm256_dpwssd_epi32(sumi, s3, _mm256_shuffle_epi32(bsums, 0xaa)); ++ sumi = _mm256_dpwssd_epi32(sumi, s4, _mm256_shuffle_epi32(bsums, 0xff)); ++ auto d8 = _mm256_set1_ps(q8.scale(iy, ibl)); ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(m4, d8), _mm256_cvtepi32_ps(sumi), acc[iy]); ++#else ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s1, _mm256_shuffle_epi32(bsums, 0x00))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s2, _mm256_shuffle_epi32(bsums, 0x55))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s3, _mm256_shuffle_epi32(bsums, 0xaa))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s4, _mm256_shuffle_epi32(bsums, 0xff))); ++ auto d8 = _mm256_set1_ps(q8.scale(iy, ibl)); ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(m4, d8), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ if constexpr (nrc_y == 1) { ++ d4 = _mm256_mul_ps(d4, d8); ++ } ++#endif ++ } ++ } ++ all_scales1 = _mm256_and_si256(all_scales1, mxf); ++ all_scales2 = _mm256_and_si256(all_scales2, mxf); ++ _mm256_storeu_si256((__m256i *)scales+0, all_scales1); ++ _mm256_storeu_si256((__m256i *)scales+1, all_scales2); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ auto iscales = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(scales + 8*ib))); ++#ifndef HAVE_FANCY_SIMD ++ auto scales = _mm256_mul_ps(d4, _mm256_cvtepi32_ps(iscales)); ++#endif ++ auto lb = _mm256_loadu_si256((const __m256i *)iq2[ibl].qs+ib); ++ qx[0] = _mm256_and_si256(lb, m03); ++ qx[1] = _mm256_and_si256(_mm256_srli_epi16(lb, 2), m03); ++ qx[2] = _mm256_and_si256(_mm256_srli_epi16(lb, 4), m03); ++ qx[3] = _mm256_and_si256(_mm256_srli_epi16(lb, 6), m03); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi)); ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ // Quants are in 0...3, so we can add add up all of them as int16_t without overflowing ++ auto sumi = _mm256_madd_epi16(m1, _mm256_add_epi16(sumi1, sumi2)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(scales, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#endif ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4y = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))); ++ acc[iy] = _mm256_fmadd_ps(d4y, _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ isum[iy] = _mm256_setzero_si256(); ++ } ++#endif ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ acc[iy] = _mm256_setzero_ps(); ++ info.store(ix+0, iy, sum); ++ } ++ } ++} ++ ++template ++static void mul_mat_q3_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ auto m30 = _mm256_set1_epi8(0x30); ++ auto m32 = _mm256_set1_epi8(32); ++ auto m03 = _mm256_set1_epi8(0x03); ++ auto m04 = _mm256_set1_epi8(0x04); ++ static const uint8_t k_shuff[32] = {0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15}; ++ auto shuff = _mm256_loadu_si256((const __m256i *)k_shuff); ++#ifdef HAVE_FANCY_SIMD ++ __m256i isum[nrc_y]; ++#elif !defined(HAVE_VNNI256) ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nbl = n / QK_K; ++ __m256 acc[nrc_y] = {}; ++ __m256i qx[4]; ++ int8_t scales[64]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q3_k_r4 * iq3 = (const block_q3_k_r4 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dl = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq3[ibl].d)); ++ auto d4 = _mm256_set_m128(dl, dl); ++#ifndef HAVE_FANCY_SIMD ++ if constexpr (nrc_y == 1) { ++ d4 = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(0, ibl))); ++ } ++#endif ++ auto slb = _mm256_loadu_si256((const __m256i *)iq3[ibl].scales_l); ++ auto shbits = _mm_loadu_si128((const __m128i *)iq3[ibl].scales_h); ++ auto shb = MM256_SET_M128I(_mm_srli_epi16(shbits, 2), shbits); ++ auto scales1 = _mm256_sub_epi8(_mm256_or_si256(_mm256_and_si256(slb, m4), _mm256_and_si256(_mm256_slli_epi16(shb, 4), m30)), m32); ++ auto scales2 = _mm256_sub_epi8(_mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(slb, 4), m4), _mm256_and_si256(shb, m30)), m32); ++ _mm256_storeu_si256((__m256i *)scales+0, scales1); ++ _mm256_storeu_si256((__m256i *)scales+1, scales2); ++ { ++#ifndef HAVE_FANCY_SIMD ++ auto min = _mm256_mul_ps(d4, _mm256_set1_ps(-4.f)); ++#endif ++ auto t1 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales1, 0)), shuff); // blocks 0, 1, 2, 3 for each row ++ auto t2 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales1, 1)), shuff); // blocks 4, 5, 6, 7 for each row ++ auto t3 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales2, 0)), shuff); // blocks 8, 9, 10, 11 for each row ++ auto t4 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm256_extracti128_si256(scales2, 1)), shuff); // blocks 12, 13, 14, 15 for each row ++ auto s1 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 0), _mm256_extracti128_si256(t1, 0)); // blocks 0, 1, 8, 9 ++ auto s2 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 1), _mm256_extracti128_si256(t1, 1)); // blocks 2, 3, 10, 11 ++ auto s3 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 0), _mm256_extracti128_si256(t2, 0)); // blocks 4, 5, 12, 13 ++ auto s4 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 1), _mm256_extracti128_si256(t2, 1)); // blocks 6, 7, 14, 15 ++#ifdef HAVE_FANCY_SIMD ++ s1 = _mm256_mullo_epi16(s1, _mm256_set1_epi16(-4)); ++ s2 = _mm256_mullo_epi16(s2, _mm256_set1_epi16(-4)); ++ s3 = _mm256_mullo_epi16(s3, _mm256_set1_epi16(-4)); ++ s4 = _mm256_mullo_epi16(s4, _mm256_set1_epi16(-4)); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums(iy, ibl); ++ auto sumi = _mm256_setzero_si256(); ++#ifdef HAVE_FANCY_SIMD ++ sumi = _mm256_dpwssd_epi32(sumi, s1, _mm256_shuffle_epi32(bsums, 0x00)); ++ sumi = _mm256_dpwssd_epi32(sumi, s2, _mm256_shuffle_epi32(bsums, 0x55)); ++ sumi = _mm256_dpwssd_epi32(sumi, s3, _mm256_shuffle_epi32(bsums, 0xaa)); ++ sumi = _mm256_dpwssd_epi32(sumi, s4, _mm256_shuffle_epi32(bsums, 0xff)); ++ isum[iy] = sumi; ++#elif defined(HAVE_VNNI256) ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s1, _mm256_shuffle_epi32(bsums, 0x00)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s2, _mm256_shuffle_epi32(bsums, 0x55)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s3, _mm256_shuffle_epi32(bsums, 0xaa)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s4, _mm256_shuffle_epi32(bsums, 0xff)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(min, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(min, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#else ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s1, _mm256_shuffle_epi32(bsums, 0x00))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s2, _mm256_shuffle_epi32(bsums, 0x55))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s3, _mm256_shuffle_epi32(bsums, 0xaa))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s4, _mm256_shuffle_epi32(bsums, 0xff))); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(min, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(min, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#endif ++ } ++ } ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ auto iscales = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(scales + 8*ib))); ++#ifndef HAVE_FANCY_SIMD ++ auto scales = _mm256_mul_ps(d4, _mm256_cvtepi32_ps(iscales)); ++#endif ++ auto lb = _mm256_loadu_si256((const __m256i *)iq3[ibl].qs+ib); ++ auto hbits = _mm_loadu_si128((const __m128i *)iq3[ibl].qh+ib); ++ auto hb = MM256_SET_M128I(hbits, _mm_slli_epi16(hbits, 4)); ++ qx[0] = _mm256_or_si256(_mm256_and_si256(lb, m03), _mm256_and_si256(m04, _mm256_srli_epi16(hb, 2))); ++ qx[1] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lb, 2), m03), _mm256_and_si256(m04, _mm256_srli_epi16(hb, 3))); ++ qx[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lb, 4), m03), _mm256_and_si256(m04, _mm256_srli_epi16(hb, 4))); ++ qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lb, 6), m03), _mm256_and_si256(m04, _mm256_srli_epi16(hb, 5))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi)); ++#elif defined(HAVE_VNNI256) ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(scales, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ // Quants are in 0...8, so we can add add up all of them as int16_t without overflowing ++ auto sumi = _mm256_madd_epi16(m1, _mm256_add_epi16(sumi1, sumi2)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(scales, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#endif ++ ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4y = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))); ++ acc[iy] = _mm256_fmadd_ps(d4y, _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ } ++#endif ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ acc[iy] = _mm256_setzero_ps(); ++ info.store(ix+0, iy, sum); ++ } ++ } ++} ++ ++template ++inline void process_min_r4_b32(int ibl, __m256 m4, __m256i mins, const Q8& q8, __m256 * acc) { ++ auto mins_l = _mm256_castsi256_si128(mins); ++ auto mins_h = _mm256_extracti128_si256(mins, 1); ++ auto aux1 = _mm_unpacklo_epi32(mins_l, mins_h); ++ auto aux2 = _mm_unpackhi_epi32(mins_l, mins_h); ++ auto ic1 = _mm256_cvtepi8_epi32(aux1); ++ auto ic2 = _mm256_cvtepi8_epi32(_mm_shuffle_epi32(aux1, 0xee)); ++ auto ic3 = _mm256_cvtepi8_epi32(aux2); ++ auto ic4 = _mm256_cvtepi8_epi32(_mm_shuffle_epi32(aux2, 0xee)); ++ if constexpr (nrc_y == 1) { ++ auto bs = _mm256_loadu_ps((const float *)q8.y[0][ibl].bsums); ++ auto sumf = _mm256_mul_ps(_mm256_cvtepi32_ps(ic1), _mm256_shuffle_ps(bs, bs, 0x00)); ++ sumf = _mm256_fmadd_ps(_mm256_cvtepi32_ps(ic2), _mm256_shuffle_ps(bs, bs, 0x55), sumf); ++ sumf = _mm256_fmadd_ps(_mm256_cvtepi32_ps(ic3), _mm256_shuffle_ps(bs, bs, 0xaa), sumf); ++ sumf = _mm256_fmadd_ps(_mm256_cvtepi32_ps(ic4), _mm256_shuffle_ps(bs, bs, 0xff), sumf); ++ acc[0] = _mm256_fmadd_ps(m4, sumf, acc[0]); ++ } else { ++ auto c1 = _mm256_mul_ps(m4, _mm256_cvtepi32_ps(ic1)); ++ auto c2 = _mm256_mul_ps(m4, _mm256_cvtepi32_ps(ic2)); ++ auto c3 = _mm256_mul_ps(m4, _mm256_cvtepi32_ps(ic3)); ++ auto c4 = _mm256_mul_ps(m4, _mm256_cvtepi32_ps(ic4)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bs = _mm256_loadu_ps((const float *)q8.y[iy][ibl].bsums); ++ acc[iy] = _mm256_fmadd_ps(c1, _mm256_shuffle_ps(bs, bs, 0x00), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(c2, _mm256_shuffle_ps(bs, bs, 0x55), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(c3, _mm256_shuffle_ps(bs, bs, 0xaa), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(c4, _mm256_shuffle_ps(bs, bs, 0xff), acc[iy]); ++ } ++ } ++} ++ ++template ++static void mul_mat_q4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = _mm256_set1_epi8(0xf); ++ auto m3 = _mm256_set1_epi8(0x30); ++ int nbl = n / QK_K; ++ union { __m256i vec; uint32_t val[8]; } hd; ++ __m256 acc[nrc_y] = {}; ++ __m256i isum[nrc_y] = {}; ++ __m256i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q4_k_r4 * iq4 = (const block_q4_k_r4 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dl = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ibl].d)); ++ auto d4 = _mm256_set_m128(_mm256_castps256_ps128(dl), _mm256_castps256_ps128(dl)); ++ auto m4 = _mm256_mul_ps(_mm256_set1_ps(-1.0f), _mm256_set_m128(_mm256_extractf128_ps(dl, 1), _mm256_extractf128_ps(dl, 1))); ++ auto lbits = _mm256_loadu_si256((const __m256i *)iq4[ibl].scales_l); ++ auto hbits128 = _mm_loadu_si128((const __m128i *)iq4[ibl].scales_h); ++ auto hbits = MM256_SET_M128I(hbits128, _mm_slli_epi16(hbits128, 4)); ++ hd.vec = _mm256_or_si256(_mm256_and_si256(lbits, mf), _mm256_and_si256(hbits, m3)); ++ auto mins = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits, 4), mf), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m3)); ++ process_min_r4_b32(ibl, m4, mins, q8, acc); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++#ifdef HAVE_VNNI256 ++ auto scales_d = _mm256_cvtepi8_epi32(_mm_set1_epi32(hd.val[ib])); ++#else ++ auto aux = _mm_set1_epi32(hd.val[ib]); ++ aux = _mm_cvtepu8_epi16(_mm_unpacklo_epi8(aux, aux)); ++ auto scales_d = MM256_SET_M128I(aux, aux); ++#endif ++ auto bits1 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+2*ib+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+2*ib+1); ++ qx[0] = _mm256_and_si256(bits1, mf); ++ qx[1] = _mm256_and_si256(bits2, mf); ++ qx[2] = _mm256_and_si256(_mm256_srli_epi16(bits1, 4), mf); ++ qx[3] = _mm256_and_si256(_mm256_srli_epi16(bits2, 4), mf); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++#ifdef HAVE_VNNI256 ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(scales_d, sumi)); ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_madd_epi16(scales_d, _mm256_add_epi16(sumi1, sumi2))); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ isum[iy] = _mm256_setzero_si256(); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ acc[iy] = _mm256_setzero_ps(); ++ info.store(ix+0, iy, sum); ++ } ++ } ++} ++ ++template ++static void mul_mat_q5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = _mm256_set1_epi8(0xf); ++ auto m10 = _mm256_set1_epi8(0x10); ++ auto m30 = _mm256_set1_epi8(0x30); ++ int nbl = n / QK_K; ++ union { __m256i vec; uint32_t val[8]; } hd; ++ __m256 acc[nrc_y] = {}; ++ __m256i isum[nrc_y] = {}; ++ __m256i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q5_k_r4 * iq5 = (const block_q5_k_r4 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dl = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq5[ibl].d)); ++ auto d4 = _mm256_set_m128(_mm256_castps256_ps128(dl), _mm256_castps256_ps128(dl)); ++ auto m4 = _mm256_mul_ps(_mm256_set1_ps(-1.0f), _mm256_set_m128(_mm256_extractf128_ps(dl, 1), _mm256_extractf128_ps(dl, 1))); ++ auto lbits = _mm256_loadu_si256((const __m256i *)iq5[ibl].scales_l); ++ auto hbits128 = _mm_loadu_si128((const __m128i *)iq5[ibl].scales_h); ++ auto hbits = MM256_SET_M128I(hbits128, _mm_slli_epi16(hbits128, 4)); ++ hd.vec = _mm256_or_si256(_mm256_and_si256(lbits, mf), _mm256_and_si256(hbits, m30)); ++ auto mins = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits, 4), mf), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m30)); ++ process_min_r4_b32(ibl, m4, mins, q8, acc); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++#ifdef HAVE_VNNI256 ++ auto scales_d = _mm256_cvtepi8_epi32(_mm_set1_epi32(hd.val[ib])); ++#else ++ auto aux = _mm_set1_epi32(hd.val[ib]); ++ aux = _mm_cvtepu8_epi16(_mm_unpacklo_epi8(aux, aux)); ++ auto scales_d = MM256_SET_M128I(aux, aux); ++#endif ++ auto lbits1 = _mm256_loadu_si256((const __m256i *)iq5[ibl].qs+2*ib+0); ++ auto lbits2 = _mm256_loadu_si256((const __m256i *)iq5[ibl].qs+2*ib+1); ++ auto hbits128 = _mm_loadu_si128((const __m128i*)iq5[ibl].qh + ib); ++ auto hbits = MM256_SET_M128I(hbits128, _mm_slli_epi16(hbits128, 4)); ++ qx[0] = _mm256_or_si256(_mm256_and_si256(lbits1, mf), _mm256_and_si256(m10, hbits)); ++ qx[1] = _mm256_or_si256(_mm256_and_si256(lbits2, mf), _mm256_and_si256(m10, _mm256_srli_epi16(hbits, 2))); ++ qx[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits1, 4), mf), _mm256_and_si256(m10, _mm256_srli_epi16(hbits, 1))); ++ qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits2, 4), mf), _mm256_and_si256(m10, _mm256_srli_epi16(hbits, 3))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++#ifdef HAVE_VNNI256 ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(scales_d, sumi)); ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ // To avoid overflow, we can only add up to 4 q5 x q8 products. ++ auto sumi = _mm256_add_epi32(_mm256_madd_epi16(scales_d, sumi1), _mm256_madd_epi16(scales_d, sumi2)); ++ isum[iy] = _mm256_add_epi32(isum[iy], sumi); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ isum[iy] = _mm256_setzero_si256(); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ acc[iy] = _mm256_setzero_ps(); ++ info.store(ix+0, iy, sum); ++ } ++ } ++} ++ ++template ++static void mul_mat_q6_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ auto m3 = _mm256_set1_epi8(0x30); ++ static const uint8_t k_shuff[32] = {0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15}; ++ auto shuff = _mm256_loadu_si256((const __m256i *)k_shuff); ++#ifdef HAVE_FANCY_SIMD ++ __m256i isum[nrc_y]; ++#elif !defined(HAVE_VNNI256) ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nbl = n / QK_K; ++ __m256 acc[nrc_y] = {}; ++ __m256i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q6_k_r4 * iq6 = (const block_q6_k_r4 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto dl = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq6[ibl].d)); ++ auto d4 = _mm256_set_m128(dl, dl); ++#ifndef HAVE_FANCY_SIMD ++ if constexpr (nrc_y == 1) { ++ d4 = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(0, ibl))); ++ } ++#endif ++ { ++#ifndef HAVE_FANCY_SIMD ++ auto min = _mm256_mul_ps(d4, _mm256_set1_ps(-32.f)); ++#endif ++ auto t1 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i *)iq6[ibl].scales+0)), shuff); // blocks 0, 1, 2, 3 for each row ++ auto t2 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i *)iq6[ibl].scales+1)), shuff); // blocks 4, 5, 6, 7 for each row ++ auto t3 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i *)iq6[ibl].scales+2)), shuff); // blocks 8, 9, 10, 11 for each row ++ auto t4 = _mm256_shuffle_epi8(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i *)iq6[ibl].scales+3)), shuff); // blocks 12, 13, 14, 15 for each row ++ auto s1 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 0), _mm256_extracti128_si256(t1, 0)); // blocks 0, 1, 8, 9 ++ auto s2 = MM256_SET_M128I(_mm256_extracti128_si256(t3, 1), _mm256_extracti128_si256(t1, 1)); // blocks 2, 3, 10, 11 ++ auto s3 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 0), _mm256_extracti128_si256(t2, 0)); // blocks 4, 5, 12, 13 ++ auto s4 = MM256_SET_M128I(_mm256_extracti128_si256(t4, 1), _mm256_extracti128_si256(t2, 1)); // blocks 6, 7, 14, 15 ++#ifdef HAVE_FANCY_SIMD ++ s1 = _mm256_mullo_epi16(s1, _mm256_set1_epi16(-32)); ++ s2 = _mm256_mullo_epi16(s2, _mm256_set1_epi16(-32)); ++ s3 = _mm256_mullo_epi16(s3, _mm256_set1_epi16(-32)); ++ s4 = _mm256_mullo_epi16(s4, _mm256_set1_epi16(-32)); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto bsums = q8.load_bsums(iy, ibl); ++ auto sumi = _mm256_setzero_si256(); ++#ifdef HAVE_FANCY_SIMD ++ sumi = _mm256_dpwssd_epi32(sumi, s1, _mm256_shuffle_epi32(bsums, 0x00)); ++ sumi = _mm256_dpwssd_epi32(sumi, s2, _mm256_shuffle_epi32(bsums, 0x55)); ++ sumi = _mm256_dpwssd_epi32(sumi, s3, _mm256_shuffle_epi32(bsums, 0xaa)); ++ sumi = _mm256_dpwssd_epi32(sumi, s4, _mm256_shuffle_epi32(bsums, 0xff)); ++ isum[iy] = sumi; ++#elif defined(HAVE_VNNI256) ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s1, _mm256_shuffle_epi32(bsums, 0x00)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s2, _mm256_shuffle_epi32(bsums, 0x55)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s3, _mm256_shuffle_epi32(bsums, 0xaa)); ++ sumi = ggml_mm256_dpwssd_epi32(sumi, s4, _mm256_shuffle_epi32(bsums, 0xff)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(min, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(min, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#else ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s1, _mm256_shuffle_epi32(bsums, 0x00))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s2, _mm256_shuffle_epi32(bsums, 0x55))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s3, _mm256_shuffle_epi32(bsums, 0xaa))); ++ sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(s4, _mm256_shuffle_epi32(bsums, 0xff))); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(min, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(min, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#endif ++ } ++ } ++ const uint32_t * scales = (const uint32_t *)iq6[ibl].scales; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ auto iscales = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i *)(scales + 2*ib))); ++#ifndef HAVE_FANCY_SIMD ++ auto scales = _mm256_mul_ps(d4, _mm256_cvtepi32_ps(iscales)); ++#endif ++ auto lbits1 = _mm256_loadu_si256((const __m256i *)iq6[ibl].ql+2*ib+0); ++ auto lbits2 = _mm256_loadu_si256((const __m256i *)iq6[ibl].ql+2*ib+1); ++ auto hbits = _mm256_loadu_si256((const __m256i *)iq6[ibl].qh+ib); ++ qx[0] = _mm256_or_si256(_mm256_and_si256(lbits1, m4), _mm256_and_si256(m3, _mm256_slli_epi16(hbits, 4))); ++ qx[1] = _mm256_or_si256(_mm256_and_si256(lbits2, m4), _mm256_and_si256(m3, hbits)); ++ qx[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits1, 4), m4), _mm256_and_si256(m3, _mm256_slli_epi16(hbits, 2))); ++ qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits2, 4), m4), _mm256_and_si256(m3, _mm256_srli_epi16(hbits, 2))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi)); ++#elif defined(HAVE_VNNI256) ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(scales, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ // Quants are in 0...63, so we can add at most 4 as int16_t to be sure of no int16_t overflow ++ auto sumi = _mm256_add_epi32(_mm256_madd_epi16(m1, sumi1), _mm256_madd_epi16(m1, sumi2)); ++ if constexpr (nrc_y == 1) { ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } else { ++ acc[iy] = _mm256_fmadd_ps(_mm256_mul_ps(scales, _mm256_set1_ps(q8.scale(iy, ibl))), _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++#endif ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4y = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))); ++ acc[iy] = _mm256_fmadd_ps(d4y, _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ } ++#endif ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ acc[iy] = _mm256_setzero_ps(); ++ info.store(ix+0, iy, sum); ++ } ++ } ++} ++ ++template void set_functions(std::array& funcs) { ++#ifdef HAVE_FANCY_SIMD ++ if constexpr (std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_iqX_k_q8_K_AVX512, Dequantizer, funcs) ++ } else { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_AVX512, Dequantizer, funcs) ++ funcs[0] = mul_mat_qX_K_q8_K_AVX512_1; ++ } ++#else ++ if constexpr (std::is_same_v || ++ std::is_same_v || ++ std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qY_K_q8_K_T, Dequantizer, funcs) ++ } else { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, Dequantizer, funcs) ++ } ++#endif ++} ++ ++// The HAVE_FANCY_SIMD should only be #if defined(__AVX512_VNNI__ && defined(__AVX512VL__) ++template ++static void mul_mat_q8_k_r8_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++#ifndef HAVE_VNNI256 ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nbl = n / QK_K; ++ __m256 acc[nrc_y] = {}; ++ __m256i isum[nrc_y] = {}; ++ __m256i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_k_r8 * iq8 = (const block_q8_k_r8 *)((const char *)vx + (ix+0)*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto d4 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[ibl].d)); ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ qx[0] = _mm256_loadu_si256((const __m256i *)iq8[ibl].qs+4*ib+0); ++ qx[1] = _mm256_loadu_si256((const __m256i *)iq8[ibl].qs+4*ib+1); ++ qx[2] = _mm256_loadu_si256((const __m256i *)iq8[ibl].qs+4*ib+2); ++ qx[3] = _mm256_loadu_si256((const __m256i *)iq8[ibl].qs+4*ib+3); ++ auto s0 = _mm256_sign_epi8(qx[0], qx[0]); ++ auto s1 = _mm256_sign_epi8(qx[1], qx[1]); ++ auto s2 = _mm256_sign_epi8(qx[2], qx[2]); ++ auto s3 = _mm256_sign_epi8(qx[3], qx[3]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+ib); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_VNNI256 ++ isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s0, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s3, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++#else ++ auto sumi1 = _mm256_madd_epi16(m1, _mm256_maddubs_epi16(s0, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0]))); ++ auto sumi2 = _mm256_madd_epi16(m1, _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1]))); ++ auto sumi3 = _mm256_madd_epi16(m1, _mm256_maddubs_epi16(s2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2]))); ++ auto sumi4 = _mm256_madd_epi16(m1, _mm256_maddubs_epi16(s3, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3]))); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_add_epi32(sumi1, sumi2)); ++ isum[iy] = _mm256_add_epi32(isum[iy], _mm256_add_epi32(sumi3, sumi4)); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4y = _mm256_mul_ps(d4, _mm256_set1_ps(q8.scale(iy, ibl))); ++ acc[iy] = _mm256_fmadd_ps(d4y, _mm256_cvtepi32_ps(isum[iy]), acc[iy]); ++ isum[iy] = _mm256_setzero_si256(); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q8_k_r16_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%16 == 0); ++ Q8 q8(info); ++ int nbl = n / QK_K; ++ __m512 acc[nrc_y] = {}; ++ __m512i isum[nrc_y] = {}; ++ __m512i qx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 16) { ++ const block_q8_k_r16 * iq16 = (const block_q8_k_r16 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { // Block of 256 ++ auto d4 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)iq16[ibl].d)); ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ qx[0] = _mm512_loadu_si512((const __m512i *)iq16[ibl].qs+4*ib+0); ++ qx[1] = _mm512_loadu_si512((const __m512i *)iq16[ibl].qs+4*ib+1); ++ qx[2] = _mm512_loadu_si512((const __m512i *)iq16[ibl].qs+4*ib+2); ++ qx[3] = _mm512_loadu_si512((const __m512i *)iq16[ibl].qs+4*ib+3); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+ib); ++ auto y256 = MM256_SET_M128I(y128, y128); ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); ++ isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ } ++ } ++ auto m4 = _mm512_mul_ps(d4, _mm512_set1_ps(-128.f)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d4y = _mm512_mul_ps(d4, _mm512_set1_ps(q8.scale(iy, ibl))); ++ acc[iy] = _mm512_fmadd_ps(d4y, _mm512_cvtepi32_ps(isum[iy]), acc[iy]); ++ acc[iy] = _mm512_fmadd_ps(m4, _mm512_set1_ps(q8.y[iy][ibl].sum), acc[iy]); ++ isum[iy] = _mm512_setzero_si512(); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm512_setzero_ps(); ++ } ++ } ++} ++#endif ++ ++template ++static void mul_mat_q8_KV_q8_KV(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ GGML_ASSERT(n%32 == 0); ++ __m256i qx[4]; ++#ifndef HAVE_FANCY_SIMD ++ __m256i sx[4]; ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ __m256i acc[nrc_y] = {}; ++ float dy[nrc_y]; ++#ifdef HAVE_FANCY_SIMD ++ int32_t sy[nrc_y]; ++#endif ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++#ifdef HAVE_FANCY_SIMD ++ auto iptr = (const int32_t *)(dptr + 1); ++ sy[iy] = -127*iptr[0]; ++#endif ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ const int8_t * q8x[4]; ++ float dx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ for (int kx = 0; kx < 4; ++kx) { ++ auto dptr = (const float *)((const char *)vx + (ix+kx)*bx); ++ dx[kx] = dptr[0]; ++ q8x[kx] = (const int8_t *)(dptr + 2); ++ } ++ for (int i = 0; i < n/32; ++i) { ++ for (int kx = 0; kx < 4; ++kx) qx[kx] = _mm256_loadu_si256((const __m256i *)q8x[kx] + i); ++ auto t0 = _mm256_unpacklo_epi32(qx[0], qx[1]); ++ auto t1 = _mm256_unpacklo_epi32(qx[2], qx[3]); ++ auto t2 = _mm256_unpackhi_epi32(qx[0], qx[1]); ++ auto t3 = _mm256_unpackhi_epi32(qx[2], qx[3]); ++#ifdef HAVE_FANCY_SIMD ++ qx[0] = _mm256_add_epi8(_mm256_unpacklo_epi64(t0, t1), _mm256_set1_epi8(127)); ++ qx[1] = _mm256_add_epi8(_mm256_unpackhi_epi64(t0, t1), _mm256_set1_epi8(127)); ++ qx[2] = _mm256_add_epi8(_mm256_unpacklo_epi64(t2, t3), _mm256_set1_epi8(127)); ++ qx[3] = _mm256_add_epi8(_mm256_unpackhi_epi64(t2, t3), _mm256_set1_epi8(127)); ++#else ++ qx[0] = _mm256_unpacklo_epi64(t0, t1); sx[0] = _mm256_sign_epi8(qx[0], qx[0]); ++ qx[1] = _mm256_unpackhi_epi64(t0, t1); sx[1] = _mm256_sign_epi8(qx[1], qx[1]); ++ qx[2] = _mm256_unpacklo_epi64(t2, t3); sx[2] = _mm256_sign_epi8(qx[2], qx[2]); ++ qx[3] = _mm256_unpackhi_epi64(t2, t3); sx[3] = _mm256_sign_epi8(qx[3], qx[3]); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = _mm256_loadu_si256((const __m256i *)q8y[iy] + i); ++#ifdef HAVE_FANCY_SIMD ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[3], _mm256_shuffle_epi32(y, 0xff)); ++#else ++ auto dot1 = _mm256_maddubs_epi16(sx[0], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ auto dot2 = _mm256_maddubs_epi16(sx[1], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ auto dot3 = _mm256_maddubs_epi16(sx[2], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ auto dot4 = _mm256_maddubs_epi16(sx[3], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++ auto dot12 = _mm256_add_epi32(_mm256_madd_epi16(m1, dot1), _mm256_madd_epi16(m1, dot2)); ++ auto dot34 = _mm256_add_epi32(_mm256_madd_epi16(m1, dot3), _mm256_madd_epi16(m1, dot4)); ++ acc[iy] = _mm256_add_epi32(acc[iy], _mm256_add_epi32(dot12, dot34)); ++#endif ++ } ++ } ++ auto scales_x = _mm_loadu_ps(dx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = _mm_add_epi32(_mm256_castsi256_si128(acc[iy]), _mm256_extracti128_si256(acc[iy], 1)); ++#ifdef HAVE_FANCY_SIMD ++ sumi = _mm_add_epi32(sumi, _mm_set1_epi32(sy[iy])); ++#endif ++ auto scale = _mm_mul_ps(scales_x, _mm_set1_ps(dy[iy])); ++ info.store(ix, iy, _mm_mul_ps(scale, _mm_cvtepi32_ps(sumi))); ++ acc[iy] = _mm256_setzero_si256(); ++ } ++ } ++} ++ ++// The HAVE_FANCY_SIMD should only be #if defined(__AVX512_VNNI__ && defined(__AVX512VL__) ++template ++static void mul_mat_q8_KV_r8_q8_KV(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%32 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++#ifndef HAVE_FANCY_SIMD ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nb = n / 16; ++ __m256i acc[nrc_y] = {}; ++ __m256i qx[4]; ++ float dy[nrc_y]; ++#ifdef HAVE_FANCY_SIMD ++ float sy[nrc_y]; ++#endif ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++#ifdef HAVE_FANCY_SIMD ++ auto iptr = (const int32_t *)(dptr + 1); ++ sy[iy] = -127*iptr[0]; ++#endif ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ auto dptr = (const float *)((const char *)vx + ix*bx); ++ auto dx = _mm256_loadu_ps(dptr); ++ auto q8x = (const int8_t *)(dptr + 8); ++ for (int ib = 0; ib < nb; ++ib) { // Blocks of 16 for 8 interleaved rows ++ qx[0] = _mm256_loadu_si256((const __m256i *)q8x+4*ib+0); ++ qx[1] = _mm256_loadu_si256((const __m256i *)q8x+4*ib+1); ++ qx[2] = _mm256_loadu_si256((const __m256i *)q8x+4*ib+2); ++ qx[3] = _mm256_loadu_si256((const __m256i *)q8x+4*ib+3); ++#ifndef HAVE_FANCY_SIMD ++ auto s0 = _mm256_sign_epi8(qx[0], qx[0]); ++ auto s1 = _mm256_sign_epi8(qx[1], qx[1]); ++ auto s2 = _mm256_sign_epi8(qx[2], qx[2]); ++ auto s3 = _mm256_sign_epi8(qx[3], qx[3]); ++#else ++ qx[0] = _mm256_add_epi8(qx[0], _mm256_set1_epi8(127)); ++ qx[1] = _mm256_add_epi8(qx[1], _mm256_set1_epi8(127)); ++ qx[2] = _mm256_add_epi8(qx[2], _mm256_set1_epi8(127)); ++ qx[3] = _mm256_add_epi8(qx[3], _mm256_set1_epi8(127)); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)q8y[iy]+ib); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_FANCY_SIMD ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[3], _mm256_shuffle_epi32(y, 0xff)); ++#else ++ auto sumi1 = _mm256_maddubs_epi16(s0, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ auto sumi2 = _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ auto sumi3 = _mm256_maddubs_epi16(s2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ auto sumi4 = _mm256_maddubs_epi16(s3, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++ auto sumi12 = _mm256_add_epi32(_mm256_madd_epi16(m1, sumi1), _mm256_madd_epi16(m1, sumi2)); ++ auto sumi34 = _mm256_add_epi32(_mm256_madd_epi16(m1, sumi3), _mm256_madd_epi16(m1, sumi4)); ++ acc[iy] = _mm256_add_epi32(acc[iy], _mm256_add_epi32(sumi12, sumi34)); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scale = _mm256_mul_ps(dx, _mm256_set1_ps(dy[iy])); ++#ifdef HAVE_FANCY_SIMD ++ acc[iy] = _mm256_add_epi32(acc[iy], _mm256_set1_epi32(sy[iy])); ++#endif ++ info.store(ix, iy, _mm256_mul_ps(scale, _mm256_cvtepi32_ps(acc[iy]))); ++ acc[iy] = _mm256_setzero_si256(); ++ } ++ } ++} ++ ++typedef struct { ++ ggml_half d[16]; ++ int8_t qs[8*QK8_1]; ++} block_q8_1_r8; ++ ++void iqk_convert_q2_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++#ifdef HAVE_FANCY_SIMD ++ constexpr int k_nr = 16; ++ using block_q8_k_r = block_q8_k_r16; ++#else ++ constexpr int k_nr = 8; ++ using block_q8_k_r = block_q8_k_r8; ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%k_nr == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q2_K * x8[k_nr]; ++ ++ block_q8_k_r * y = (block_q8_k_r *)vy; ++ ++ float f_values[QK_K]; ++ uint32_t block[8]; ++ ++ __m256i xv[4]; ++ ++ auto ml = _mm256_set1_epi8(0x03); ++ auto sign_bit = _mm256_set1_ps(-0.0f); ++ auto perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ ++ for (int ix = 0; ix < nrc_x; ix += k_nr) { ++ for (int k = 0; k < k_nr; ++k) x8[k] = (const block_q2_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < k_nr; ++k) { ++ auto vd = _mm256_set1_ps(GGML_FP16_TO_FP32(x8[k][i].d)); ++ auto vm = _mm256_mul_ps(_mm256_set1_ps(GGML_FP16_TO_FP32(x8[k][i].dmin)), _mm256_set1_ps(-1.f)); ++ auto block_max = _mm256_setzero_ps(); ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto bits = _mm256_loadu_si256((const __m256i *)x8[k][i].qs+i128); ++ xv[0] = _mm256_and_si256(bits, ml); ++ xv[1] = _mm256_and_si256(_mm256_srli_epi16(bits, 2), ml); ++ xv[2] = _mm256_and_si256(_mm256_srli_epi16(bits, 4), ml); ++ xv[3] = _mm256_and_si256(_mm256_srli_epi16(bits, 6), ml); ++ for (int l = 0; l < 4; ++l) { ++ auto q1 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(xv[l])); ++ auto q2 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xv[l], 1)); ++ q1 = _mm256_mullo_epi16(q1, _mm256_set1_epi16(x8[k][i].scales[8*i128 + 2*l + 0] & 0xf)); ++ q2 = _mm256_mullo_epi16(q2, _mm256_set1_epi16(x8[k][i].scales[8*i128 + 2*l + 1] & 0xf)); ++ auto m1 = _mm256_mul_ps(vm, _mm256_set1_ps(x8[k][i].scales[8*i128 + 2*l + 0] >> 4)); ++ auto m2 = _mm256_mul_ps(vm, _mm256_set1_ps(x8[k][i].scales[8*i128 + 2*l + 1] >> 4)); ++ auto v0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q1))), vd, m1); ++ auto v1 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q1, 1))), vd, m1); ++ auto v2 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_castsi256_si128(q2))), vd, m2); ++ auto v3 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm256_extracti128_si256(q2, 1))), vd, m2); ++ auto max = _mm256_max_ps(_mm256_max_ps(_mm256_andnot_ps(sign_bit, v0), _mm256_andnot_ps(sign_bit, v1)), ++ _mm256_max_ps(_mm256_andnot_ps(sign_bit, v2), _mm256_andnot_ps(sign_bit, v3))); ++ block_max = _mm256_max_ps(block_max, max); ++ _mm256_storeu_ps(f_values + 128*i128 + 32*l + 0, v0); ++ _mm256_storeu_ps(f_values + 128*i128 + 32*l + 8, v1); ++ _mm256_storeu_ps(f_values + 128*i128 + 32*l + 16, v2); ++ _mm256_storeu_ps(f_values + 128*i128 + 32*l + 24, v3); ++ } ++ } ++ auto max4 = _mm_max_ps(_mm256_extractf128_ps(block_max, 1), _mm256_castps256_ps128(block_max)); ++ max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); ++ max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4)); ++ float d = _mm_cvtss_f32(max4)/127.f; ++ auto id = _mm256_set1_ps(d != 0.0f ? 1/d : 0.0f); ++ y[i].d[k] = GGML_FP32_TO_FP16(d); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto v0 = _mm256_loadu_ps(f_values + 32*ib32 + 0); ++ auto v1 = _mm256_loadu_ps(f_values + 32*ib32 + 8); ++ auto v2 = _mm256_loadu_ps(f_values + 32*ib32 + 16); ++ auto v3 = _mm256_loadu_ps(f_values + 32*ib32 + 24); ++ auto i0 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(v0, id), _MM_ROUND_NEAREST)); ++ auto i1 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(v1, id), _MM_ROUND_NEAREST)); ++ auto i2 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(v2, id), _MM_ROUND_NEAREST)); ++ auto i3 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(v3, id), _MM_ROUND_NEAREST)); ++ i0 = _mm256_packs_epi32(i0, i1); ++ i2 = _mm256_packs_epi32(i2, i3); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, perm); ++ ++ _mm256_storeu_si256((__m256i *)block, i0); ++ auto q8 = (uint32_t *)y[i].qs + 8*k_nr*ib32; ++ for (int l = 0; l < 8; ++l) { ++ q8[k_nr*l + k] = block[l]; ++ } ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int l = 0; l < 64; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[i].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[i].qs + l, v); ++ } ++#endif ++ } ++ y += nb; ++ } ++} ++ ++void iqk_convert_q4_k_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q4_K * x8[8]; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ ggml_half dh[16]; ++ uint16_t all_ls[128]; ++ ++ uint32_t utmp[4]; ++ const uint8_t * u8 = (const uint8_t *)utmp; ++ uint32_t block[8]; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q4_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ dh[k+0] = x8[k][i].d; ++ dh[k+8] = x8[k][i].dmin; ++ make_q4_scales(x8[k][i].scales, utmp); ++ auto qs = x8[k][i].qs; ++ for (int ib64 = 0; ib64 < 4; ++ib64) { ++ all_ls[8*(2*ib64 + 0) + k ] = u8[2*ib64+0]; ++ all_ls[8*(2*ib64 + 1) + k ] = u8[2*ib64+1]; ++ all_ls[8*(2*ib64 + 0) + k + 64] = u8[2*ib64+8]; ++ all_ls[8*(2*ib64 + 1) + k + 64] = u8[2*ib64+9]; ++ auto bits = _mm256_loadu_si256((const __m256i *)qs+ib64); ++ auto values1 = _mm256_and_si256(bits, _mm256_set1_epi8(0xf)); ++ auto values2 = _mm256_and_si256(_mm256_srli_epi16(bits, 4), _mm256_set1_epi8(0xf)); ++ _mm256_storeu_si256((__m256i *)block, values1); ++ auto q8 = (uint32_t *)y[2*ib64+0].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ _mm256_storeu_si256((__m256i *)block, values2); ++ q8 = (uint32_t *)y[2*ib64+1].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ auto vd = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)dh+0)); ++ auto vm = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)dh+1)); ++ vm = _mm256_mul_ps(_mm256_set1_ps(-1.f), vm); ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ auto iscales16 = _mm_loadu_si128((const __m128i *)all_ls + ib32); ++ auto iscales32 = _mm256_cvtepi16_epi32(iscales16); ++ auto scales = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(iscales32)); ++ _mm_storeu_si128((__m128i *)y[ib32].d+0, _mm256_cvtps_ph(scales, _MM_FROUND_TO_NEAREST_INT)); ++ iscales16 = _mm_loadu_si128((const __m128i *)all_ls + ib32 + 8); ++ iscales32 = _mm256_cvtepi16_epi32(iscales16); ++ scales = _mm256_mul_ps(vm, _mm256_cvtepi32_ps(iscales32)); ++ _mm_storeu_si128((__m128i *)y[ib32].d+1, _mm256_cvtps_ph(scales, _MM_FROUND_TO_NEAREST_INT)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++void iqk_convert_q5_k_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q5_K * x8[8]; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ ggml_half dh[16]; ++ uint16_t all_ls[128]; ++ ++ uint32_t utmp[4]; ++ const uint8_t * u8 = (const uint8_t *)utmp; ++ uint32_t block[8]; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q5_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ dh[k+0] = x8[k][i].d; ++ dh[k+8] = x8[k][i].dmin; ++ make_q4_scales(x8[k][i].scales, utmp); ++ auto qs = x8[k][i].qs; ++ auto hbits = _mm256_loadu_si256((const __m256i *)x8[k][i].qh); ++ for (int ib64 = 0; ib64 < 4; ++ib64) { ++ all_ls[8*(2*ib64 + 0) + k ] = u8[2*ib64+0]; ++ all_ls[8*(2*ib64 + 1) + k ] = u8[2*ib64+1]; ++ all_ls[8*(2*ib64 + 0) + k + 64] = u8[2*ib64+8]; ++ all_ls[8*(2*ib64 + 1) + k + 64] = u8[2*ib64+9]; ++ auto bits = _mm256_loadu_si256((const __m256i *)qs+ib64); ++ auto values1 = _mm256_and_si256(bits, _mm256_set1_epi8(0xf)); ++ auto values2 = _mm256_and_si256(_mm256_srli_epi16(bits, 4), _mm256_set1_epi8(0xf)); ++ values1 = _mm256_or_si256(values1, _mm256_and_si256(_mm256_set1_epi8(0x10), _mm256_slli_epi16(hbits, 4))); ++ values2 = _mm256_or_si256(values2, _mm256_and_si256(_mm256_set1_epi8(0x10), _mm256_slli_epi16(hbits, 3))); ++ hbits = _mm256_srli_epi16(hbits, 2); ++ _mm256_storeu_si256((__m256i *)block, values1); ++ auto q8 = (uint32_t *)y[2*ib64+0].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ _mm256_storeu_si256((__m256i *)block, values2); ++ q8 = (uint32_t *)y[2*ib64+1].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ auto vd = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)dh+0)); ++ auto vm = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)dh+1)); ++ vm = _mm256_mul_ps(_mm256_set1_ps(-1.f), vm); ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ auto iscales16 = _mm_loadu_si128((const __m128i *)all_ls + ib32); ++ auto iscales32 = _mm256_cvtepi16_epi32(iscales16); ++ auto scales = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(iscales32)); ++ _mm_storeu_si128((__m128i *)y[ib32].d+0, _mm256_cvtps_ph(scales, _MM_FROUND_TO_NEAREST_INT)); ++ iscales16 = _mm_loadu_si128((const __m128i *)all_ls + ib32 + 8); ++ iscales32 = _mm256_cvtepi16_epi32(iscales16); ++ scales = _mm256_mul_ps(vm, _mm256_cvtepi32_ps(iscales32)); ++ _mm_storeu_si128((__m128i *)y[ib32].d+1, _mm256_cvtps_ph(scales, _MM_FROUND_TO_NEAREST_INT)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++void iqk_convert_q6_k_q8_0_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q6_K * x8[8]; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ float all_s[64]; ++ uint32_t block[8]; ++ __m256i values[8]; ++ ++ auto ml = _mm256_set1_epi8(0x0f); ++ auto mh = _mm256_set1_epi8(0x30); ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q6_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ auto ql = x8[k][i].ql; ++ auto qh = x8[k][i].qh; ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto lbits1 = _mm256_loadu_si256((const __m256i *)ql + 2*i128 + 0); ++ auto lbits2 = _mm256_loadu_si256((const __m256i *)ql + 2*i128 + 1); ++ auto hbits = _mm256_loadu_si256((const __m256i *)qh + i128); ++ values[4*i128+0] = _mm256_or_si256(_mm256_and_si256(lbits1, ml), _mm256_and_si256(_mm256_slli_epi16(hbits, 4), mh)); ++ values[4*i128+1] = _mm256_or_si256(_mm256_and_si256(lbits2, ml), _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ values[4*i128+2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits1, 4), ml), _mm256_and_si256(hbits, mh)); ++ values[4*i128+3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits2, 4), ml), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), mh)); ++ } ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ // We have two blocks of 16 with different scales ++ // We multiply the quants with the scales, find the max value, and convert to 8-bit quants with a single block scale. ++ auto q8 = _mm256_add_epi8(values[ib32], _mm256_set1_epi8(-32)); ++ auto q16_l = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(q8)); ++ auto q16_h = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(q8, 1)); ++ q16_l = _mm256_mullo_epi16(q16_l, _mm256_set1_epi16(x8[k][i].scales[2*ib32+0])); ++ q16_h = _mm256_mullo_epi16(q16_h, _mm256_set1_epi16(x8[k][i].scales[2*ib32+1])); ++ auto abs_q16_l = _mm256_sign_epi16(q16_l, q16_l); ++ auto abs_q16_h = _mm256_sign_epi16(q16_h, q16_h); ++ auto max_q16 = _mm256_max_epi16(abs_q16_l, abs_q16_h); ++ auto max_q32 = _mm256_cvtepi16_epi32(_mm_max_epi16(_mm256_castsi256_si128(max_q16), _mm256_extracti128_si256(max_q16, 1))); ++ auto imax4 = _mm_max_epi32(_mm256_castsi256_si128(max_q32), _mm256_extracti128_si256(max_q32, 1)); ++ auto max4 = _mm_cvtepi32_ps(imax4); ++ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); ++ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); ++ float max = _mm_cvtss_f32(max4) / 127; ++ all_s[8*ib32+k] = d*max; ++ if (max > 1e-9f) { ++ auto scale = _mm256_set1_ps(1/max); ++ auto i0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_l)); ++ auto i1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_l, 1)); ++ auto i2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_h)); ++ auto i3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_h, 1)); ++ i0 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i0)), _MM_ROUND_NEAREST)); ++ i1 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i1)), _MM_ROUND_NEAREST)); ++ i2 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i2)), _MM_ROUND_NEAREST)); ++ i3 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i3)), _MM_ROUND_NEAREST)); ++ i0 = _mm256_packs_epi32(i0, i1); ++ i2 = _mm256_packs_epi32(i2, i3); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)); ++ _mm256_storeu_si256((__m256i *)block, i0); ++ } else { ++ _mm256_storeu_si256((__m256i *)block, _mm256_setzero_si256()); ++ } ++ auto qs = (uint32_t *)y[ib32].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ _mm_storeu_si128((__m128i *)y[ib32].d, _mm256_cvtps_ph(_mm256_loadu_ps(all_s + 8*ib32), _MM_FROUND_TO_NEAREST_INT)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++void iqk_convert_q3_k_q8_0_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q3_K * x8[8]; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ float all_s[64]; ++ uint32_t block[8]; ++ __m256i values[8]; ++ ++ ScaleQ3 sc3; ++ auto ml = _mm256_set1_epi8(0x03); ++ auto mh = _mm256_set1_epi8(0x04); ++ ++ union { __m256i vec; int16_t val[16]; } helper; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q3_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ auto hbits = _mm256_loadu_si256((const __m256i *)x8[k][i].hmask); ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto q2bits = _mm256_loadu_si256((const __m256i *)x8[k][i].qs + i128); ++ values[4*i128+0] = _mm256_and_si256(q2bits, ml); ++ values[4*i128+1] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), ml); ++ values[4*i128+2] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), ml); ++ values[4*i128+3] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), ml); ++ values[4*i128+0] = _mm256_or_si256(values[4*i128+0], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ values[4*i128+1] = _mm256_or_si256(values[4*i128+1], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ values[4*i128+2] = _mm256_or_si256(values[4*i128+2], _mm256_and_si256(hbits, mh)); ++ values[4*i128+3] = _mm256_or_si256(values[4*i128+3], _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh)); ++ values[4*i128+0] = _mm256_sub_epi8(values[4*i128+0], mh); ++ values[4*i128+1] = _mm256_sub_epi8(values[4*i128+1], mh); ++ values[4*i128+2] = _mm256_sub_epi8(values[4*i128+2], mh); ++ values[4*i128+3] = _mm256_sub_epi8(values[4*i128+3], mh); ++ hbits = _mm256_srli_epi16(hbits, 4); ++ } ++ helper.vec = _mm256_cvtepi8_epi16(sc3.make_scales((const uint16_t *)x8[k][i].scales)); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto q16_l = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(values[ib32])); ++ auto q16_h = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(values[ib32], 1)); ++ q16_l = _mm256_mullo_epi16(q16_l, _mm256_set1_epi16(helper.val[2*ib32+0])); ++ q16_h = _mm256_mullo_epi16(q16_h, _mm256_set1_epi16(helper.val[2*ib32+1])); ++ auto abs_q16_l = _mm256_sign_epi16(q16_l, q16_l); ++ auto abs_q16_h = _mm256_sign_epi16(q16_h, q16_h); ++ auto max_q16 = _mm256_max_epi16(abs_q16_l, abs_q16_h); ++ auto max_q32 = _mm256_cvtepi16_epi32(_mm_max_epi16(_mm256_castsi256_si128(max_q16), _mm256_extracti128_si256(max_q16, 1))); ++ auto imax4 = _mm_max_epi32(_mm256_castsi256_si128(max_q32), _mm256_extracti128_si256(max_q32, 1)); ++ auto max4 = _mm_cvtepi32_ps(imax4); ++ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); ++ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); ++ float max = _mm_cvtss_f32(max4) / 127; ++ all_s[8*ib32+k] = d*max; ++ if (max > 1e-9f) { ++ auto scale = _mm256_set1_ps(1/max); ++ auto i0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_l)); ++ auto i1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_l, 1)); ++ auto i2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_h)); ++ auto i3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_h, 1)); ++ i0 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i0)), _MM_ROUND_NEAREST)); ++ i1 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i1)), _MM_ROUND_NEAREST)); ++ i2 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i2)), _MM_ROUND_NEAREST)); ++ i3 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i3)), _MM_ROUND_NEAREST)); ++ i0 = _mm256_packs_epi32(i0, i1); ++ i2 = _mm256_packs_epi32(i2, i3); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)); ++ _mm256_storeu_si256((__m256i *)block, i0); ++ } else { ++ _mm256_storeu_si256((__m256i *)block, _mm256_setzero_si256()); ++ } ++ auto qs = (uint32_t *)y[ib32].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ _mm_storeu_si128((__m128i *)y[ib32].d, _mm256_cvtps_ph(_mm256_loadu_ps(all_s + 8*ib32), _MM_FROUND_TO_NEAREST_INT)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++void iqk_convert_q3_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++#ifdef HAVE_FANCY_SIMD ++ constexpr int k_nr = 16; ++ using block_q8_k_r = block_q8_k_r16; ++#else ++ constexpr int k_nr = 8; ++ using block_q8_k_r = block_q8_k_r8; ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%k_nr == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q3_K * x8[k_nr]; ++ ++ block_q8_k_r * y = (block_q8_k_r *)vy; ++ ++ uint32_t block[8]; ++ __m256i values[8]; ++ ++ ScaleQ3 sc3; ++ auto ml = _mm256_set1_epi8(0x03); ++ auto mh = _mm256_set1_epi8(0x04); ++ ++ union { __m256i vec; int16_t val[16]; } helper; ++ ++ for (int ix = 0; ix < nrc_x; ix += k_nr) { ++ for (int k = 0; k < k_nr; ++k) x8[k] = (const block_q3_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < k_nr; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ auto hbits = _mm256_loadu_si256((const __m256i *)x8[k][i].hmask); ++ helper.vec = _mm256_cvtepi8_epi16(sc3.make_scales((const uint16_t *)x8[k][i].scales)); ++ auto max_i16 = _mm256_setzero_si256(); ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto q2bits = _mm256_loadu_si256((const __m256i *)x8[k][i].qs + i128); ++ values[4*i128+0] = _mm256_and_si256(q2bits, ml); ++ values[4*i128+1] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), ml); ++ values[4*i128+2] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), ml); ++ values[4*i128+3] = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), ml); ++ values[4*i128+0] = _mm256_or_si256(values[4*i128+0], _mm256_and_si256(_mm256_slli_epi16(hbits, 2), mh)); ++ values[4*i128+1] = _mm256_or_si256(values[4*i128+1], _mm256_and_si256(_mm256_slli_epi16(hbits, 1), mh)); ++ values[4*i128+2] = _mm256_or_si256(values[4*i128+2], _mm256_and_si256(hbits, mh)); ++ values[4*i128+3] = _mm256_or_si256(values[4*i128+3], _mm256_and_si256(_mm256_srli_epi16(hbits, 1), mh)); ++ values[4*i128+0] = _mm256_sub_epi8(values[4*i128+0], mh); ++ values[4*i128+1] = _mm256_sub_epi8(values[4*i128+1], mh); ++ values[4*i128+2] = _mm256_sub_epi8(values[4*i128+2], mh); ++ values[4*i128+3] = _mm256_sub_epi8(values[4*i128+3], mh); ++ hbits = _mm256_srli_epi16(hbits, 4); ++ ++ for (int l = 0; l < 4; ++l) { ++ auto q16_l = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(values[4*i128+l])); ++ auto q16_h = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(values[4*i128+l], 1)); ++ q16_l = _mm256_mullo_epi16(_mm256_set1_epi16(helper.val[8*i128+2*l+0]), q16_l); ++ q16_h = _mm256_mullo_epi16(_mm256_set1_epi16(helper.val[8*i128+2*l+1]), q16_h); ++ max_i16 = _mm256_max_epi16(max_i16, _mm256_sign_epi16(q16_l, q16_l)); ++ max_i16 = _mm256_max_epi16(max_i16, _mm256_sign_epi16(q16_h, q16_h)); ++ } ++ } ++ auto max_q32 = _mm256_cvtepi16_epi32(_mm_max_epi16(_mm256_castsi256_si128(max_i16), _mm256_extracti128_si256(max_i16, 1))); ++ auto imax4 = _mm_max_epi32(_mm256_castsi256_si128(max_q32), _mm256_extracti128_si256(max_q32, 1)); ++ auto max4 = _mm_cvtepi32_ps(imax4); ++ max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); ++ max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4)); ++ bool needs_scaling = true; ++ float dnew = _mm_cvtss_f32(max4) / 127; ++ if (dnew < 1.f) { ++ dnew = 1.f; needs_scaling = false; ++ } ++ d *= dnew; ++ y[i].d[k] = GGML_FP32_TO_FP16(d); ++ auto scale = _mm256_set1_ps(std::abs(dnew) > 1e-9f ? 1/dnew : 0.f); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto q16_l = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(values[ib32])); ++ auto q16_h = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(values[ib32], 1)); ++ q16_l = _mm256_mullo_epi16(q16_l, _mm256_set1_epi16(helper.val[2*ib32+0])); ++ q16_h = _mm256_mullo_epi16(q16_h, _mm256_set1_epi16(helper.val[2*ib32+1])); ++ if (needs_scaling) { ++ auto i0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_l)); ++ auto i1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_l, 1)); ++ auto i2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(q16_h)); ++ auto i3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(q16_h, 1)); ++ i0 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i0)), _MM_ROUND_NEAREST)); ++ i1 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i1)), _MM_ROUND_NEAREST)); ++ i2 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i2)), _MM_ROUND_NEAREST)); ++ i3 = _mm256_cvtps_epi32(_mm256_round_ps(_mm256_mul_ps(scale, _mm256_cvtepi32_ps(i3)), _MM_ROUND_NEAREST)); ++ i0 = _mm256_packs_epi32(i0, i1); ++ i2 = _mm256_packs_epi32(i2, i3); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)); ++ _mm256_storeu_si256((__m256i *)block, i0); ++ } else { ++ // 0, 1, 2, 3, 4, 5, 6, 7, 8, 16, 17, 18, 19, 20, 21, 22, 23, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 ++ auto i0 = _mm256_packs_epi16(q16_l, q16_h); ++ auto i0_l = _mm256_castsi256_si128(i0); ++ auto i0_h = _mm256_extracti128_si256(i0, 1); ++ _mm_storeu_si128((__m128i *)block+0, _mm_unpacklo_epi64(i0_l, i0_h)); ++ _mm_storeu_si128((__m128i *)block+1, _mm_unpackhi_epi64(i0_l, i0_h)); ++ } ++ auto qs = (uint32_t *)y[i].qs + 8*k_nr*ib32; ++ for (int l = 0; l < 8; ++l) { ++ qs[k_nr*l + k] = block[l]; ++ } ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int l = 0; l < 64; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[i].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[i].qs + l, v); ++ } ++#endif ++ } ++ y += nb; ++ } ++} ++ ++// TODO: move this to iqk_gemm_iquants ++void iqk_convert_iq4_xs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ ++#ifdef HAVE_FANCY_SIMD ++ constexpr int k_nr = 16; ++ using block_q8_k_r = block_q8_k_r16; ++#else ++ constexpr int k_nr = 8; ++ using block_q8_k_r = block_q8_k_r8; ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%k_nr == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_iq4_xs * x8[k_nr]; ++ ++ block_q8_k_r * y = (block_q8_k_r *)vy; ++ ++ auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); ++ auto values = MM256_SET_M128I(values128, values128); ++ ++ int16_t ls[16]; ++ float dnew[k_nr]; ++ uint32_t block[8]; ++ __m256i xv[8]; ++ ++ for (int ix = 0; ix < nrc_x; ix += k_nr) { ++ for (int k = 0; k < k_nr; ++k) x8[k] = (const block_iq4_xs *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < k_nr; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ ls[2*ib32+0] = ls[2*ib32+1] = (((x8[k][i].scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((x8[k][i].scales_h >> 2*ib32) & 3) << 4)) - 32; ++ auto bits = _mm_loadu_si128((const __m128i *)x8[k][i].qs + ib32); ++ xv[ib32] = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(bits, 4), bits), _mm256_set1_epi8(0xf)); ++ xv[ib32] = _mm256_shuffle_epi8(values, xv[ib32]); ++ } ++ dnew[k] = d * convert_to_q8_k_r8(k, 1.f/127, xv, ls, block, y[i].qs); ++ } ++#ifdef HAVE_FANCY_SIMD ++ _mm256_storeu_si256((__m256i *)y[i].d, _mm512_cvtps_ph(_mm512_loadu_ps(dnew), _MM_ROUND_NEAREST)); ++ for (int l = 0; l < 64; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[i].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[i].qs + l, v); ++ } ++#else ++ _mm_storeu_si128((__m128i *)y[i].d, _mm256_cvtps_ph(_mm256_loadu_ps(dnew), _MM_ROUND_NEAREST)); ++#endif ++ } ++ y += nb; ++ } ++} ++ ++ ++} // namespace ++ ++bool iqk_set_kernels_kquants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16) { ++ ++ auto etypeA = ggml_type(typeA); ++ auto expected_type_B = etypeA == GGML_TYPE_IQ4_XS_R8 || etypeA == GGML_TYPE_Q4_K_R4 || etypeA == GGML_TYPE_Q5_K_R4 ? GGML_TYPE_Q8_K32 ++ //: etypeA == GGML_TYPE_Q8_K_R8 ? GGML_TYPE_Q8_KR8 ++ : etypeA == GGML_TYPE_Q8_KV || etypeA == GGML_TYPE_Q8_KV_R8 ? GGML_TYPE_Q8_KV ++ : etypeA == GGML_TYPE_Q4_K || etypeA == GGML_TYPE_Q5_K || ++ etypeA == GGML_TYPE_Q6_K ? GGML_TYPE_Q8_2_X4 ++ //etypeA == GGML_TYPE_Q6_K || etypeA == GGML_TYPE_Q3_K ? GGML_TYPE_Q8_2_X4 ++ //: etypeA == GGML_TYPE_Q4_K || etypeA == GGML_TYPE_Q5_K ? GGML_TYPE_Q8_2_X4 ++ : GGML_TYPE_Q8_K; ++ ++ if (ne00%QK_K != 0 || ggml_type(typeB) != expected_type_B) { ++ return false; ++ } ++ ++ func16 = nullptr; ++ ++ switch (typeA) { ++ case GGML_TYPE_Q2_K: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q3_K: ++ set_functions(kernels); ++ //IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qY_K_q8_2_X4_T, DequantizerQ3K_AVX2, kernels); ++ break; ++ case GGML_TYPE_Q4_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_2_X4_T, DequantizerQ4K_AVX2, kernels); ++ //set_functions(kernels); ++ break; ++ case GGML_TYPE_Q5_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_2_X4_T, DequantizerQ5K_AVX2, kernels); ++ //set_functions(kernels); ++ break; ++ case GGML_TYPE_Q6_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qY_K_q8_2_X4_T, DequantizerQ6K_AVX2, kernels); ++ //set_functions(kernels); ++ break; ++ case GGML_TYPE_IQ4_XS: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q2_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q2_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q3_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q3_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q4_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q4_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q5_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q5_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q6_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q6_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_IQ4_XS_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_iq4_xs_r8_q8_k_avx2, kernels) ++ break; ++ case GGML_TYPE_Q8_K_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_k_r8_q8_k, kernels) ++//#ifdef HAVE_FANCY_SIMD ++// func16 = mul_mat_q8_k_r8_q8_k<16>; ++//#endif ++ break; ++#ifdef HAVE_FANCY_SIMD ++ case GGML_TYPE_Q8_K_R16: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_k_r16_q8_k, kernels) ++ break; ++#endif ++ case GGML_TYPE_Q8_KV: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_KV_q8_KV, kernels) ++#ifdef HAVE_FANCY_SIMD ++ func16 = mul_mat_q8_KV_q8_KV<16>; ++#endif ++ break; ++ case GGML_TYPE_Q8_KV_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_KV_r8_q8_KV, kernels); ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++ ++} ++ ++bool iqk_convert_kquants_q8X_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ switch (ggml_type(type)) { ++ case GGML_TYPE_Q2_K: iqk_convert_q2_k_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q3_K: iqk_convert_q3_k_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q4_K: iqk_convert_q4_k_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_K: iqk_convert_q5_k_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q6_K: iqk_convert_q6_k_q8_0_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_IQ4_XS: iqk_convert_iq4_xs_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ default: return false; ++ } ++ return true; ++} ++ ++#else ++// --------------------------------- __aarch64__ -------------------------------------- ++ ++namespace { ++ ++template ++inline void accum_mins_8(const int16x8_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8s = q8.load_bsums8(iy, i); ++ int32x4_t b1 = vmull_s16(vget_low_s16(mins), vget_low_s16(q8s)); ++ int32x4_t b2 = vmull_s16(vget_high_s16(mins), vget_high_s16(q8s)); ++ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(b1, b2)); ++ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i))); ++ } ++} ++template ++inline void accum_mins_16(const int16x8x2_t& mins, const Q8& q8, float32x4_t * acc, int i, float c) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8s = q8.load_bsums(iy, i); ++ int32x4_t b1 = vmull_s16(vget_low_s16 (mins.val[0]), vget_low_s16 (q8s.val[0])); ++ int32x4_t b2 = vmull_s16(vget_high_s16(mins.val[0]), vget_high_s16(q8s.val[0])); ++ int32x4_t b3 = vmull_s16(vget_low_s16 (mins.val[1]), vget_low_s16 (q8s.val[1])); ++ int32x4_t b4 = vmull_s16(vget_high_s16(mins.val[1]), vget_high_s16(q8s.val[1])); ++ float32x4_t prod = vcvtq_f32_s32(vaddq_s32(vaddq_s32(b1, b2), vaddq_s32(b3, b4))); ++ acc[iy] = vmlaq_f32(acc[iy], prod, vdupq_n_f32(c*q8.scale(iy, i))); ++ } ++} ++ ++struct Scales8 { ++ uint32_t utmp[4]; ++ const uint8_t * sc8 = (const uint8_t *)utmp; ++ template ++ inline int32x4x2_t process_scales_mins(const Qx& x, const Q8& q8, int i, float32x4_t * acc) { ++ make_q4_scales(x.scales, utmp); ++ int16x8_t mins = vmovl_s8(vld1_s8((const int8_t *)sc8 + 8)); ++ accum_mins_8(mins, q8, acc, i, -GGML_FP16_TO_FP32(x.dmin)); ++ ++ uint8x8_t scales8 = vld1_u8(sc8); ++ uint16x8_t scales16 = vmovl_u8(scales8); ++ int32x4x2_t scales = {vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales16))), ++ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales16)))}; ++ return scales; ++ } ++ inline float32x4x4_t make_scales(float d, float m, const uint8_t * scales) { ++ make_q4_scales(scales, utmp); ++ auto d16 = vmovl_u8(vld1_u8(sc8+0)); ++ auto m16 = vmovl_u8(vld1_u8(sc8+8)); ++ auto vd = vdupq_n_f32(d); ++ auto vm = vdupq_n_f32(m); ++ return { vmulq_f32(vd, vcvtq_f32_u32(vmovl_u16(vget_low_u16 (d16)))), ++ vmulq_f32(vd, vcvtq_f32_u32(vmovl_u16(vget_high_u16(d16)))), ++ vmulq_f32(vm, vcvtq_f32_u32(vmovl_u16(vget_low_u16 (m16)))), ++ vmulq_f32(vm, vcvtq_f32_u32(vmovl_u16(vget_high_u16(m16)))) }; ++ } ++}; ++ ++struct DequantizerQ4K final : public BaseDequantizer { ++ DequantizerQ4K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ return s8.process_scales_mins(x[i], q8, i, acc); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ float m = -GGML_FP16_TO_FP32(x[i].dmin); ++ return s8.make_scales(d, m, x[i].scales); ++ } ++ inline void prepare(int i, int j) { ++ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j); ++ else bits.prepare(x[i].qs+64*j); ++ } ++ ++ Q4bits bits; ++ Scales8 s8; ++ ++}; ++ ++struct HighBit5 { ++ const uint8x16_t mhb = vdupq_n_u8(0x10); ++ uint8x16x2_t bits; ++ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) { ++ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 4), mhb)); ++ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 4), mhb)); ++ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 3), mhb)); ++ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 3), mhb)); ++ ++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb)); ++ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb)); ++ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb)); ++ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb)); ++ ++ if (do_shift) { ++ bits.val[0] = vshrq_n_u8(bits.val[0], 4); ++ bits.val[1] = vshrq_n_u8(bits.val[1], 4); ++ } ++ } ++}; ++ ++struct HighBit3 { ++ const uint8x16_t mhb = vdupq_n_u8(0x04); ++ uint8x16x2_t bits; ++ inline void apply(uint8x16x4_t& b1, uint8x16x4_t& b2, bool do_shift) { ++ b1.val[0] = vorrq_u8(b1.val[0], vandq_u8(vshlq_n_u8(bits.val[0], 2), mhb)); ++ b1.val[1] = vorrq_u8(b1.val[1], vandq_u8(vshlq_n_u8(bits.val[1], 2), mhb)); ++ b1.val[2] = vorrq_u8(b1.val[2], vandq_u8(vshlq_n_u8(bits.val[0], 1), mhb)); ++ b1.val[3] = vorrq_u8(b1.val[3], vandq_u8(vshlq_n_u8(bits.val[1], 1), mhb)); ++ ++ b2.val[0] = vorrq_u8(b2.val[0], vandq_u8(bits.val[0], mhb)); ++ b2.val[1] = vorrq_u8(b2.val[1], vandq_u8(bits.val[1], mhb)); ++ b2.val[2] = vorrq_u8(b2.val[2], vandq_u8(vshrq_n_u8(bits.val[0], 1), mhb)); ++ b2.val[3] = vorrq_u8(b2.val[3], vandq_u8(vshrq_n_u8(bits.val[1], 1), mhb)); ++ ++ if (do_shift) { ++ bits.val[0] = vshrq_n_u8(bits.val[0], 4); ++ bits.val[1] = vshrq_n_u8(bits.val[1], 4); ++ } ++ } ++}; ++ ++struct DequantizerQ5K final : public BaseDequantizer { ++ DequantizerQ5K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ h.bits = vld1q_u8_x2(x[i].qh); ++ return s8.process_scales_mins(x[i], q8, i, acc); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ float m = -GGML_FP16_TO_FP32(x[i].dmin); ++ h.bits = vld1q_u8_x2(x[i].qh); ++ return s8.make_scales(d, m, x[i].scales); ++ } ++ inline void prepare(int i, int j) { ++ if (nrc == 1) bits.prepare_v2(x[i].qs+64*j); ++ else bits.prepare(x[i].qs+64*j); ++ h.apply(bits.b1, bits.b2, j == 0); ++ } ++ ++ Q4bits bits; ++ HighBit5 h; ++ Scales8 s8; ++ ++ uint8x16x2_t hbits; ++ ++}; ++ ++inline int32x4x4_t make_wider(const int16x8x2_t& scales16) { ++ int32x4x4_t scales = { ++ vmovl_s16(vget_low_s16 (scales16.val[0])), ++ vmovl_s16(vget_high_s16(scales16.val[0])), ++ vmovl_s16(vget_low_s16 (scales16.val[1])), ++ vmovl_s16(vget_high_s16(scales16.val[1])), ++ }; ++ return scales; ++} ++ ++template ++inline int32x4x4_t process_scales_mins_16(const int8x16_t& scales8, const Q8& q8, float32x4_t * acc, int i, float c) { ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(scales8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(scales8)); ++ accum_mins_16(scales16, q8, acc, i, c); ++ return make_wider(scales16); ++} ++ ++struct DequantizerQ6K final : public BaseDequantizer { ++ DequantizerQ6K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ return process_scales_mins_16(vld1q_s8(x[i].scales), q8, acc, i, -32.f*d); ++ } ++ inline float32x4x4_t new_block(int i) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ auto vd = vdupq_n_f32(d); ++ auto scales8 = vld1q_s8(x[i].scales); ++ auto scales16_1 = vmovl_s8(vget_low_s8 (scales8)); ++ auto scales16_2 = vmovl_s8(vget_high_s8(scales8)); ++ return { vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (scales16_1)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_high_s16(scales16_1)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (scales16_2)))), ++ vmulq_f32(vd, vcvtq_f32_s32(vmovl_s16(vget_high_s16(scales16_2)))) }; ++ } ++ ++ inline void prepare(int i, int j) { ++ ++ auto hbits = vld1q_u8_x2(x[i].qh + 32*j); ++ ++ bits.prepare64(x[i].ql+64*j); ++ bits.b1.val[0] = vorrq_u8(bits.b1.val[0], vandq_u8(vshlq_n_u8(hbits.val[0], 4), mhb)); ++ bits.b1.val[1] = vorrq_u8(bits.b1.val[1], vandq_u8(vshlq_n_u8(hbits.val[1], 4), mhb)); ++ bits.b1.val[2] = vorrq_u8(bits.b1.val[2], vandq_u8(vshlq_n_u8(hbits.val[0], 2), mhb)); ++ bits.b1.val[3] = vorrq_u8(bits.b1.val[3], vandq_u8(vshlq_n_u8(hbits.val[1], 2), mhb)); ++ ++ bits.b2.val[0] = vorrq_u8(bits.b2.val[0], vandq_u8(hbits.val[0], mhb)); ++ bits.b2.val[1] = vorrq_u8(bits.b2.val[1], vandq_u8(hbits.val[1], mhb)); ++ bits.b2.val[2] = vorrq_u8(bits.b2.val[2], vandq_u8(vshrq_n_u8(hbits.val[0], 2), mhb)); ++ bits.b2.val[3] = vorrq_u8(bits.b2.val[3], vandq_u8(vshrq_n_u8(hbits.val[1], 2), mhb)); ++ ++ } ++ ++ inline void prepare_signed(int i, int j) { ++ prepare(i, j); ++ auto m32 = vdupq_n_s8(-32); ++ for (int k = 0; k < 4; ++k) { ++ bits.b1.val[k] = vaddq_s8(bits.b1.val[k], m32); ++ bits.b2.val[k] = vaddq_s8(bits.b2.val[k], m32); ++ } ++ } ++ ++ Q4bits bits; ++ ++ const uint8x16_t mhb = vdupq_n_u8(0x30); ++ ++}; ++ ++struct DequantizerQ3K final : public BaseDequantizer { ++ DequantizerQ3K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ h.bits = vld1q_u8_x2(x[i].hmask); ++ mask = vdupq_n_u8(0x01); ++ const uint16_t * sc16 = (const uint16_t *)x[i].scales; ++ uint32_t aux0 = sc16[0] | (sc16[1] << 16); ++ uint32_t aux1 = sc16[2] | (sc16[3] << 16); ++ uint32_t aux2 = sc16[4] | (sc16[5] << 16); ++ aux32[0] = (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030); ++ aux32[1] = (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030); ++ aux32[2] = ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030); ++ aux32[3] = ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030); ++ auto scales8 = vaddq_s8(vld1q_s8((const int8_t *)aux32), vdupq_n_s8(-32)); ++ if (nrc > 1) { ++ return process_scales_mins_16(scales8, q8, acc, i, -4.f*d); ++ } ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(scales8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(scales8)); ++ return make_wider(scales16); ++ } ++ ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs+32*j); ++ if (nrc > 1) { ++ h.apply(bits.b1, bits.b2, j == 0); ++ } else { ++ auto minus4 = vdupq_n_u8(0xfc); ++ auto zero = vdupq_n_u8(0); ++ bits.b1.val[0] = vorrq_u8(bits.b1.val[0], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b1.val[1] = vorrq_u8(bits.b1.val[1], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b1.val[2] = vorrq_u8(bits.b1.val[2], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b1.val[3] = vorrq_u8(bits.b1.val[3], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b2.val[0] = vorrq_u8(bits.b2.val[0], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b2.val[1] = vorrq_u8(bits.b2.val[1], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ bits.b2.val[2] = vorrq_u8(bits.b2.val[2], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[0], mask), zero))); ++ bits.b2.val[3] = vorrq_u8(bits.b2.val[3], vandq_u8(minus4, vceqq_u8(vandq_u8(h.bits.val[1], mask), zero))); ++ mask = vshlq_n_u8(mask, 1); ++ } ++ } ++ ++ uint32_t aux32[4]; ++ ++ Q2bits bits; ++ ++ uint8x16_t mask; ++ HighBit3 h; ++ ++}; ++ ++struct DequantizerQ2K final : public BaseDequantizer { ++ DequantizerQ2K(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc) {} ++ ++ constexpr static int num_blocks() { return 16; } ++ constexpr static bool should_scale_quants() { return true; } ++ ++ template ++ inline void process_scales(int i, const Q8& q8, float32x4_t * acc) { ++ d = GGML_FP16_TO_FP32(x[i].d); ++ auto scales_and_mins = vld1q_u8(x[i].scales); ++ auto mins8 = vreinterpretq_s8_u8(vshrq_n_u8(scales_and_mins, 4)); ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(mins8)); ++ scales16.val[1] = vmovl_s8(vget_high_s8(mins8)); ++ accum_mins_16(scales16, q8, acc, i, -GGML_FP16_TO_FP32(x[i].dmin)); ++ ++ scales8 = vandq_u8(scales_and_mins, vdupq_n_u8(0xf)); ++ } ++ ++ template ++ inline int32x4x4_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ process_scales(i, q8, acc); ++ int16x8x2_t scales16; ++ scales16.val[0] = vmovl_s8(vget_low_s8(vreinterpretq_s8_u8(scales8))); ++ scales16.val[1] = vmovl_s8(vget_high_s8(vreinterpretq_s8_u8(scales8))); ++ return make_wider(scales16); ++ } ++ ++ template ++ inline void compute(const Q8& q8, int i, int j, int32x4_t * sumi) { ++ auto m1 = vdupq_n_u8(1); ++ auto shuffle = vdupq_n_u8(8*j); ++ bits.b1.val[0] = vmulq_u8(bits.b1.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[1] = vmulq_u8(bits.b1.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[2] = vmulq_u8(bits.b1.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b1.val[3] = vmulq_u8(bits.b1.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[0] = vmulq_u8(bits.b2.val[0], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[1] = vmulq_u8(bits.b2.val[1], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[2] = vmulq_u8(bits.b2.val[2], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ bits.b2.val[3] = vmulq_u8(bits.b2.val[3], vqtbl1q_u8(scales8, shuffle)); shuffle = vaddq_u8(shuffle, m1); ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto q8b_1 = q8.load_quants(iy, i, 4*j+0); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[0]), q8b_1.val[0]), ++ vreinterpretq_s8_u8(bits.b1.val[1]), q8b_1.val[1]); ++ ++ auto q8b_2 = q8.load_quants(iy, i, 4*j+1); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b1.val[2]), q8b_2.val[0]), ++ vreinterpretq_s8_u8(bits.b1.val[3]), q8b_2.val[1]); ++ ++ auto q8b_3 = q8.load_quants(iy, i, 4*j+2); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[0]), q8b_3.val[0]), ++ vreinterpretq_s8_u8(bits.b2.val[1]), q8b_3.val[1]); ++ ++ auto q8b_4 = q8.load_quants(iy, i, 4*j+3); ++ sumi[iy] = ggml_vdotq_s32(ggml_vdotq_s32(sumi[iy], vreinterpretq_s8_u8(bits.b2.val[2]), q8b_4.val[0]), ++ vreinterpretq_s8_u8(bits.b2.val[3]), q8b_4.val[1]); ++ } ++ } ++ ++ inline void prepare(int i, int j) { ++ bits.prepare(x[i].qs+32*j); ++ } ++ ++ uint32_t aux32[4]; ++ ++ uint8x16_t scales8; ++ ++ Q2bits bits; ++ ++}; ++ ++struct DequantizerIQ4XS final : public BaseDequantizer { ++ ++ static int8x16_t load_values() { ++ static const int8_t iq4nl_values[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; ++ return vld1q_s8(iq4nl_values); ++ } ++ ++ DequantizerIQ4XS(const void * vx, size_t bx, int nrc) : BaseDequantizer(vx, bx, nrc), values(load_values()) {} ++ ++ constexpr static int num_blocks() { return 8; } ++ constexpr static bool should_scale_quants() { return false; } ++ ++ inline void new_row(int ix) { x = (const block_iq4_xs *)((const char *)vx + bx*ix); } ++ ++ template ++ inline int32x4x2_t new_block(int i, const Q8& q8, float32x4_t * acc) { ++ (void)q8; ++ (void)acc; ++ d = GGML_FP16_TO_FP32(x[i].d); ++ const uint16_t scales_h = x[i].scales_h; ++ const uint16_t * scales_l = (const uint16_t *)x[i].scales_l; ++ aux32[0] = scales_l[0] | (scales_l[1] << 16); ++ aux32[1] = aux32[0] >> 4; ++ // scl is ordered as 0, 2, 4, 6, 1, 3, 5, 7 ++ uint8x8_t scl8 = vand_u8(vld1_u8((const uint8_t *)aux32), vdup_n_u8(0xf)); ++ uint16_t * aux16 = (uint16_t *)aux32; ++ aux16[0] = scales_h << 4; aux16[1] = scales_h << 2; aux16[2] = scales_h; aux16[3] = scales_h >> 2; ++ // sch is ordered as 0, 4, 1, 5, 2, 6, 3, 7 ++ uint8x8_t sch8 = vand_u8(vld1_u8((const uint8_t *)aux16), vdup_n_u8(0x30)); ++ int8x8_t scales8 = vadd_s8(vreinterpret_s8_u8(vorr_u8(scl8, vtbl1_u8(sch8, vreinterpret_u8_u32(hshuff)))), vdup_n_s8(-32)); ++ // shuffle 0, 2, 4, 6, 1, 3, 5, 7 -> 0, 1, 2, 3, 4, 5, 6, 7 ++ scales8 = vtbl1_s8(scales8, vreinterpret_s8_u32(hshuff)); ++ int16x8_t scales16 = vmovl_s8(scales8); ++ int32x4x2_t scales = {vmovl_s16(vget_low_s16(scales16)), vmovl_s16(vget_high_s16(scales16))}; ++ return scales; ++ } ++ inline void prepare(int i, int j) { ++ bits.prepare16(x[i].qs+64*j); ++ //if (nrc == 1) { ++ // bits.prepare16_v2(x[i].qs+64*j); ++ //} else { ++ // bits.prepare16(x[i].qs+64*j); ++ //} ++ for (int k = 0; k < 4; ++k) { ++ bits.b1.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b1.val[k])); ++ bits.b2.val[k] = vreinterpretq_u8_s8(vqtbl1q_s8(values, bits.b2.val[k])); ++ } ++ } ++ ++ Q4bits bits; ++ const int8x16_t values; ++ uint32_t aux32[2]; ++ ++ constexpr static uint32x2_t hshuff = {0x05010400, 0x07030602}; ++ ++}; ++ ++IQK_ALWAYS_INLINE void prepare_q4_k_quants(const uint8x16_t& m4, const uint8x16x4_t& bits, int8x16_t * qx) { ++ qx[0] = vandq_u8(bits.val[0], m4); // 0...3 from the 4 rows ++ qx[1] = vandq_u8(bits.val[1], m4); // 16..19 ++ qx[2] = vandq_u8(bits.val[2], m4); // 4...7 ++ qx[3] = vandq_u8(bits.val[3], m4); // 20..23 ++ qx[4] = vshrq_n_u8(bits.val[0], 4); // 8..11 ++ qx[5] = vshrq_n_u8(bits.val[1], 4); // 24..27 ++ qx[6] = vshrq_n_u8(bits.val[2], 4); // 12..15 ++ qx[7] = vshrq_n_u8(bits.val[3], 4); // 28..31 ++} ++ ++template ++void mul_mat_q2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = vdupq_n_u8(0x0f); ++ auto m03 = vdupq_n_u8(0x03); ++ int nbl = n / QK_K; ++ int8x16_t qx[4]; ++ float32x4_t acc[nrc_y] = {}; ++ int16x8x4_t i16scales; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q2_k_r4 * iq2 = (const block_q2_k_r4 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ int32x4_t isum[nrc_y] = {}; ++ auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq2[ibl].d)); ++ auto m4 = vmulq_f32(vdupq_n_f32(-1.f), vcvt_f32_f16(vld1_f16((const float16_t *)iq2[ibl].d+4))); ++ for (int is = 0; is < 2; ++is) { ++ auto sl = vld1q_u8_x2(iq2[ibl].scales + 32*is); ++ auto m = vshrq_n_u8(sl.val[0], 4); ++ i16scales.val[0] = vmovl_u8(vget_low_u8 (m)); ++ i16scales.val[1] = vmovl_u8(vget_high_u8(m)); ++ m = vshrq_n_u8(sl.val[1], 4); ++ i16scales.val[2] = vmovl_u8(vget_low_u8 (m)); ++ i16scales.val[3] = vmovl_u8(vget_high_u8(m)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = vdupq_n_s32(0); ++ auto bsums = vld1q_s16(q8.y[iy][ibl].bsums + 8*is); ++ auto b8 = vget_low_s16(bsums); ++ //auto bsums = q8.load_bsums(iy, ibl); ++ //auto b8 = vget_low_s16(bsums.val[0]); ++ sumi = vmlal_lane_s16(sumi, vget_low_s16 (i16scales.val[0]), b8, 0); ++ sumi = vmlal_lane_s16(sumi, vget_high_s16(i16scales.val[0]), b8, 1); ++ sumi = vmlal_lane_s16(sumi, vget_low_s16 (i16scales.val[1]), b8, 2); ++ sumi = vmlal_lane_s16(sumi, vget_high_s16(i16scales.val[1]), b8, 3); ++ b8 = vget_high_s16(bsums); ++ sumi = vmlal_lane_s16(sumi, vget_low_s16 (i16scales.val[2]), b8, 0); ++ sumi = vmlal_lane_s16(sumi, vget_high_s16(i16scales.val[2]), b8, 1); ++ sumi = vmlal_lane_s16(sumi, vget_low_s16 (i16scales.val[3]), b8, 2); ++ sumi = vmlal_lane_s16(sumi, vget_high_s16(i16scales.val[3]), b8, 3); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(m4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(sumi)); ++ } ++ m = vandq_u8(sl.val[0], mf); ++ i16scales.val[0] = vmovl_u8(vget_low_u8 (m)); ++ i16scales.val[1] = vmovl_u8(vget_high_u8(m)); ++ m = vandq_u8(sl.val[1], mf); ++ i16scales.val[2] = vmovl_u8(vget_low_u8 (m)); ++ i16scales.val[3] = vmovl_u8(vget_high_u8(m)); ++ for (int ib = 0; ib < 4; ++ib) { ++ auto bits = vld1q_u8_x2(iq2[ibl].qs + 128*is + 32*ib); ++ auto scales = vmovl_s16(vget_low_s16 (i16scales.val[ib])); ++ qx[0] = vreinterpretq_s8_u8(vandq_u8( bits.val[0], m03)); ++ qx[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[0], 2), m03)); ++ qx[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[0], 4), m03)); ++ qx[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[0], 6), m03)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ scales = vmovl_s16(vget_high_s16(i16scales.val[ib])); ++ qx[0] = vreinterpretq_s8_u8(vandq_u8( bits.val[1], m03)); ++ qx[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[1], 2), m03)); ++ qx[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[1], 4), m03)); ++ qx[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(bits.val[1], 6), m03)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib+16); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(d4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_q3_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = vdupq_n_u8(0x0f); ++ auto m30 = vdupq_n_u8(0x30); ++ auto m32 = vdupq_n_s8(-32); ++ auto m03 = vdupq_n_u8(0x03); ++ auto m04 = vdupq_n_u8(0x04); ++ int nbl = n / QK_K; ++ int8x16_t qx[4]; ++ float32x4_t acc[nrc_y] = {}; ++ int8x16x4_t i8scales; ++ int16x8x4_t i16scales; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q3_k_r4 * iq3 = (const block_q3_k_r4 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ int32x4_t isum[nrc_y] = {}; ++ auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq3[ibl].d)); ++ auto sl = vld1q_u8_x2(iq3[ibl].scales_l); ++ auto sh = vld1q_u8(iq3[ibl].scales_h); ++ i8scales.val[0] = vaddq_s8(m32, vorrq_u8(vandq_u8(sl.val[0], mf), vandq_u8(vshlq_n_u8(sh, 4), m30))); ++ i8scales.val[1] = vaddq_s8(m32, vorrq_u8(vandq_u8(sl.val[1], mf), vandq_u8(vshlq_n_u8(sh, 2), m30))); ++ i8scales.val[2] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(sl.val[0], 4), vandq_u8(sh, m30))); ++ i8scales.val[3] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(sl.val[1], 4), vandq_u8(vshrq_n_u8(sh, 2), m30))); ++ for (int is = 0; is < 2; ++is) { ++ i16scales.val[0] = vmovl_s8(vget_low_s8 (i8scales.val[2*is+0])); ++ i16scales.val[1] = vmovl_s8(vget_high_s8(i8scales.val[2*is+0])); ++ i16scales.val[2] = vmovl_s8(vget_low_s8 (i8scales.val[2*is+1])); ++ i16scales.val[3] = vmovl_s8(vget_high_s8(i8scales.val[2*is+1])); ++ for (int ib = 0; ib < 4; ++ib) { ++ auto lbits = vld1q_u8_x2(iq3[ibl].qs + 128*is + 32*ib); ++ auto hbits = vld1q_u8(iq3[ibl].qh + 64*is + 16*ib); ++ hbits = veorq_u8(hbits, vdupq_n_u8(0xff)); ++ auto scales = vmovl_s16(vget_low_s16 (i16scales.val[ib])); ++ qx[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8( lbits.val[0], m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshlq_n_u8(hbits, 2)))); ++ qx[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[0], 2), m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshlq_n_u8(hbits, 1)))); ++ qx[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[0], 4), m03)), vreinterpretq_s8_u8(vandq_u8(m04, hbits))); ++ qx[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[0], 6), m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshrq_n_u8(hbits, 1)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ scales = vmovl_s16(vget_high_s16(i16scales.val[ib])); ++ qx[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8( lbits.val[1], m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshrq_n_u8(hbits, 2)))); ++ qx[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[1], 2), m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshrq_n_u8(hbits, 3)))); ++ qx[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[1], 4), m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshrq_n_u8(hbits, 4)))); ++ qx[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(lbits.val[1], 6), m03)), vreinterpretq_s8_u8(vandq_u8(m04, vshrq_n_u8(hbits, 5)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib+16); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(d4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_q4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = vdupq_n_u8(0xf); ++ auto m3 = vdupq_n_u8(0x30); ++ int nbl = n / QK_K; ++ int8x16_t qx[8]; ++ int8x16x2_t iscales; ++ int32x4x4_t scales; ++ float32x4_t acc[nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q4_k_r4 * iq4 = (const block_q4_k_r4 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq4[ibl].d)); ++ auto m4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq4[ibl].d+4)); ++ m4 = vmulq_f32(m4, vdupq_n_f32(-1.f)); ++ auto sl = vld1q_u8_x2(iq4[ibl].scales_l); ++ auto sh = vld1q_u8(iq4[ibl].scales_h); ++ iscales.val[0] = vorrq_u8(vshrq_n_u8(sl.val[0], 4), vandq_u8(vshlq_n_u8(sh, 2), m3)); ++ iscales.val[1] = vorrq_u8(vshrq_n_u8(sl.val[1], 4), vandq_u8(vshrq_n_u8(sh, 2), m3)); ++ for (int is = 0; is < 2; ++is) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[is])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[is])); ++ float32x4x4_t fscales; ++ fscales.val[0] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_low_s16(iscales16_1)))); ++ fscales.val[1] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_high_s16(iscales16_1)))); ++ fscales.val[2] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_low_s16(iscales16_2)))); ++ fscales.val[3] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_high_s16(iscales16_2)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto m8 = vld1q_f32((const float *)q8.y[iy][ibl].bsums + 4*is); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[0], m8, 0); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[1], m8, 1); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[2], m8, 2); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[3], m8, 3); ++ } ++ } ++ iscales.val[0] = vorrq_u8(vandq_u8(sl.val[0], mf), vandq_u8(vshlq_n_u8(sh, 4), m3)); ++ iscales.val[1] = vorrq_u8(vandq_u8(sl.val[1], mf), vandq_u8(sh, m3)); ++ int32x4_t isum[nrc_y] = {}; ++ for (int is = 0; is < 2; ++is) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[is])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[is])); ++ scales.val[0] = vmovl_s16(vget_low_s16(iscales16_1)); ++ scales.val[1] = vmovl_s16(vget_high_s16(iscales16_1)); ++ scales.val[2] = vmovl_s16(vget_low_s16(iscales16_2)); ++ scales.val[3] = vmovl_s16(vget_high_s16(iscales16_2)); ++ for (int ib = 0; ib < 4; ++ib) { ++ auto bits = vld1q_u8_x4(iq4[ibl].qs + 256*is + 64*ib); ++ prepare_q4_k_quants(mf, bits, qx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ibl].qs+128*is+32*ib); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales.val[ib], sumi); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(d4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_q5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = vdupq_n_u8(0xf); ++ auto m30 = vdupq_n_u8(0x30); ++ auto m10 = vdupq_n_u8(0x10); ++ int nbl = n / QK_K; ++ int8x16_t qx[8]; ++ int8x16x2_t iscales; ++ int32x4x4_t scales; ++ float32x4_t acc[nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q5_k_r4 * iq5 = (const block_q5_k_r4 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq5[ibl].d)); ++ auto m4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq5[ibl].d+4)); ++ m4 = vmulq_f32(m4, vdupq_n_f32(-1.f)); ++ auto sl = vld1q_u8_x2(iq5[ibl].scales_l); ++ auto sh = vld1q_u8(iq5[ibl].scales_h); ++ iscales.val[0] = vorrq_u8(vshrq_n_u8(sl.val[0], 4), vandq_u8(vshlq_n_u8(sh, 2), m30)); ++ iscales.val[1] = vorrq_u8(vshrq_n_u8(sl.val[1], 4), vandq_u8(vshrq_n_u8(sh, 2), m30)); ++ for (int is = 0; is < 2; ++is) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[is])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[is])); ++ float32x4x4_t fscales; ++ fscales.val[0] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_low_s16(iscales16_1)))); ++ fscales.val[1] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_high_s16(iscales16_1)))); ++ fscales.val[2] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_low_s16(iscales16_2)))); ++ fscales.val[3] = vmulq_f32(m4, vcvtq_f32_s32(vmovl_s16(vget_high_s16(iscales16_2)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto m8 = vld1q_f32((const float *)q8.y[iy][ibl].bsums + 4*is); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[0], m8, 0); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[1], m8, 1); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[2], m8, 2); ++ acc[iy] = vmlaq_laneq_f32(acc[iy], fscales.val[3], m8, 3); ++ } ++ } ++ iscales.val[0] = vorrq_u8(vandq_u8(sl.val[0], mf), vandq_u8(vshlq_n_u8(sh, 4), m30)); ++ iscales.val[1] = vorrq_u8(vandq_u8(sl.val[1], mf), vandq_u8(sh, m30)); ++ int32x4_t isum[nrc_y] = {}; ++ for (int is = 0; is < 2; ++is) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[is])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[is])); ++ scales.val[0] = vmovl_s16(vget_low_s16(iscales16_1)); ++ scales.val[1] = vmovl_s16(vget_high_s16(iscales16_1)); ++ scales.val[2] = vmovl_s16(vget_low_s16(iscales16_2)); ++ scales.val[3] = vmovl_s16(vget_high_s16(iscales16_2)); ++ for (int ib = 0; ib < 4; ++ib) { ++ auto lbits = vld1q_u8_x4(iq5[ibl].qs + 256*is + 64*ib); ++ auto hbits2 = vld1q_u8(iq5[ibl].qh + 64*is + 16*ib); ++ auto hbits1 = vshlq_n_u8(hbits2, 4); ++ prepare_q4_k_quants(mf, lbits, qx); ++ qx[0] = vorrq_u8(qx[0], vandq_u8(m10, hbits1)); ++ qx[1] = vorrq_u8(qx[1], vandq_u8(m10, hbits2)); ++ qx[2] = vorrq_u8(qx[2], vandq_u8(m10, vshrq_n_u8(hbits1, 2))); ++ qx[3] = vorrq_u8(qx[3], vandq_u8(m10, vshrq_n_u8(hbits2, 2))); ++ qx[4] = vorrq_u8(qx[4], vandq_u8(m10, vshrq_n_u8(hbits1, 1))); ++ qx[5] = vorrq_u8(qx[5], vandq_u8(m10, vshrq_n_u8(hbits2, 1))); ++ qx[6] = vorrq_u8(qx[6], vandq_u8(m10, vshrq_n_u8(hbits1, 3))); ++ qx[7] = vorrq_u8(qx[7], vandq_u8(m10, vshrq_n_u8(hbits2, 3))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ibl].qs+128*is+32*ib); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales.val[ib], sumi); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(d4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_q6_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto mf = vdupq_n_u8(0x0f); ++ auto m3 = vdupq_n_u8(0x30); ++ auto m32 = vdupq_n_s8(-32); ++ int nbl = n / QK_K; ++ int8x16_t qx[4]; ++ float32x4_t acc[nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q6_k_r4 * iq6 = (const block_q6_k_r4 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq6[ibl].d)); ++ int32x4_t isum[nrc_y] = {}; ++ for (int is = 0; is < 2; ++is) { ++ for (int ib = 0; ib < 4; ++ib) { ++ auto lbits = vld1q_u8_x4(iq6[ibl].ql + 256*is + 64*ib); ++ auto hbits = vld1q_u8(iq6[ibl].qh + 128*is + 32*ib); ++ auto iscales = vmovl_s8(vld1_s8(iq6[ibl].scales + 32*is + 8*ib)); ++ auto scales = vmovl_s16(vget_low_s16(iscales)); ++ qx[0] = vaddq_s8(m32, vorrq_u8(vandq_u8 (lbits.val[0], mf), vandq_u8(m3, vshlq_n_u8(hbits, 4)))); ++ qx[1] = vaddq_s8(m32, vorrq_u8(vandq_u8 (lbits.val[2], mf), vandq_u8(m3, hbits))); ++ qx[2] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits.val[0], 4), vandq_u8(m3, vshlq_n_u8(hbits, 2)))); ++ qx[3] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits.val[2], 4), vandq_u8(m3, vshrq_n_u8(hbits, 2)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ scales = vmovl_s16(vget_high_s16(iscales)); ++ hbits = vld1q_u8(iq6[ibl].qh + 128*is + 32*ib + 16); ++ qx[0] = vaddq_s8(m32, vorrq_u8(vandq_u8 (lbits.val[1], mf), vandq_u8(m3, vshlq_n_u8(hbits, 4)))); ++ qx[1] = vaddq_s8(m32, vorrq_u8(vandq_u8 (lbits.val[3], mf), vandq_u8(m3, hbits))); ++ qx[2] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits.val[1], 4), vandq_u8(m3, vshlq_n_u8(hbits, 2)))); ++ qx[3] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits.val[3], 4), vandq_u8(m3, vshrq_n_u8(hbits, 2)))); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib+16); ++ auto sumi = interleaved_dotq(qx, y); ++ isum[iy] = vmlaq_s32(isum[iy], scales, sumi); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(d4, vdupq_n_f32(q8.scale(iy, ibl))), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_q8_k_r8_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ int nbl = n / QK_K; ++ float32x4_t acc[2*nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_k_r8 * iq8 = (const block_q8_k_r8 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ auto d4l = vcvt_f32_f16(vld1_f16((const float16_t *)iq8[ibl].d+0)); ++ auto d4h = vcvt_f32_f16(vld1_f16((const float16_t *)iq8[ibl].d+4)); ++ int32x4_t isum[2*nrc_y] = {}; ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ auto q1 = vld1q_s8_x4(iq8[ibl].qs + 128*ib + 0); ++ auto q2 = vld1q_s8_x4(iq8[ibl].qs + 128*ib + 64); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8.y[iy][ibl].qs+16*ib); ++ isum[2*iy+0] = vdotq_laneq_s32(isum[2*iy+0], q1.val[0], y, 0); ++ isum[2*iy+1] = vdotq_laneq_s32(isum[2*iy+1], q1.val[1], y, 0); ++ isum[2*iy+0] = vdotq_laneq_s32(isum[2*iy+0], q1.val[2], y, 1); ++ isum[2*iy+1] = vdotq_laneq_s32(isum[2*iy+1], q1.val[3], y, 1); ++ isum[2*iy+0] = vdotq_laneq_s32(isum[2*iy+0], q2.val[0], y, 2); ++ isum[2*iy+1] = vdotq_laneq_s32(isum[2*iy+1], q2.val[1], y, 2); ++ isum[2*iy+0] = vdotq_laneq_s32(isum[2*iy+0], q2.val[2], y, 3); ++ isum[2*iy+1] = vdotq_laneq_s32(isum[2*iy+1], q2.val[3], y, 3); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d8 = vdupq_n_f32(q8.scale(iy, ibl)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(d4l, d8), vcvtq_f32_s32(isum[2*iy+0])); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(d4h, d8), vcvtq_f32_s32(isum[2*iy+1])); ++ // Why did I have this? It is plain wrong! ++ //const float * bsum = (const float *)q8.y[iy][ibl].bsums; ++ //auto m8 = vdupq_n_f32(-128.f*bsum[0]); ++ //acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], d4l, m8); ++ //acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], d4l, m8); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix+0, iy, acc[2*iy+0]); ++ info.store(ix+4, iy, acc[2*iy+1]); ++ acc[2*iy+0] = acc[2*iy+1] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_iq4_xs_r8_q8_k(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = vdupq_n_u8(0xf); ++ auto m3 = vdupq_n_u8(0x30); ++ auto m32 = vdupq_n_s8(-32); ++ auto values = vld1q_s8(iq4k_values); ++ int nbl = n / QK_K; ++ int8x16_t qx[8]; ++ int8x16x4_t iscales; ++ int32x4x2_t scales; ++ float32x4_t acc[2*nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_xs_r8 * iq4 = (const block_iq4_xs_r8 *)((const char *)vx + ix*bx); ++ for (int ibl = 0; ibl < nbl; ++ibl) { ++ auto d4_f16 = vld1q_f16((const float16_t *)iq4[ibl].d); ++ auto d4l = vcvt_f32_f16(vget_low_f16 (d4_f16)); ++ auto d4h = vcvt_f32_f16(vget_high_f16(d4_f16)); ++ auto sl = vld1q_u8_x2(iq4[ibl].scales_l); ++ auto sh = vld1q_u8(iq4[ibl].scales_h); ++ iscales.val[0] = vaddq_s8(vorrq_u8(vandq_u8(sl.val[0], m4), vandq_u8(vshlq_n_u8(sh, 4), m3)), m32); ++ iscales.val[1] = vaddq_s8(vorrq_u8(vandq_u8(sl.val[1], m4), vandq_u8(vshlq_n_u8(sh, 2), m3)), m32); ++ iscales.val[2] = vaddq_s8(vorrq_u8(vshrq_n_u8(sl.val[0], 4), vandq_u8(sh, m3)), m32); ++ iscales.val[3] = vaddq_s8(vorrq_u8(vshrq_n_u8(sl.val[1], 4), vandq_u8(vshrq_n_u8(sh, 2), m3)), m32); ++ int32x4_t isum[nrc_y] = {}; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[ib64])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[ib64])); ++ scales.val[0] = vmovl_s16(vget_low_s16(iscales16_1)); ++ scales.val[1] = vmovl_s16(vget_low_s16(iscales16_2)); ++ for (int l = 0; l < 2; ++l) { ++ uint8x16x2_t bits; ++ bits.val[0] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l); ++ bits.val[1] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 32); ++ prepare_iq4_nl_quants_r8(values, m4, bits, qx+0); ++ bits.val[0] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 64); ++ bits.val[1] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 96); ++ prepare_iq4_nl_quants_r8(values, m4, bits, qx+4); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ibl].qs+64*ib64+32*l); ++ auto sumi = vdupq_n_s32(0); ++ sumi = vdotq_laneq_s32(sumi, qx[0], y.val[0], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[1], y.val[0], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[2], y.val[0], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[3], y.val[0], 3); ++ sumi = vdotq_laneq_s32(sumi, qx[4], y.val[1], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[5], y.val[1], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[6], y.val[1], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[7], y.val[1], 3); ++ isum[iy] = vmlaq_s32(isum[iy], sumi, scales.val[l]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d8 = vdupq_n_f32(q8.scale(iy, ibl)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(d4l, d8), vcvtq_f32_s32(isum[iy])); ++ isum[iy] = vdupq_n_s32(0); ++ } ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ auto iscales16_1 = vmovl_s8(vget_low_s8(iscales.val[ib64])); ++ auto iscales16_2 = vmovl_s8(vget_high_s8(iscales.val[ib64])); ++ scales.val[0] = vmovl_s16(vget_high_s16(iscales16_1)); ++ scales.val[1] = vmovl_s16(vget_high_s16(iscales16_2)); ++ for (int l = 0; l < 2; ++l) { ++ uint8x16x2_t bits; ++ bits.val[0] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 16); ++ bits.val[1] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 48); ++ prepare_iq4_nl_quants_r8(values, m4, bits, qx+0); ++ bits.val[0] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l + 80); ++ bits.val[1] = vld1q_u8(iq4[ibl].qs + 256*ib64 + 128*l +112); ++ prepare_iq4_nl_quants_r8(values, m4, bits, qx+4); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ibl].qs+64*ib64+32*l); ++ auto sumi = vdupq_n_s32(0); ++ sumi = vdotq_laneq_s32(sumi, qx[0], y.val[0], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[1], y.val[0], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[2], y.val[0], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[3], y.val[0], 3); ++ sumi = vdotq_laneq_s32(sumi, qx[4], y.val[1], 0); ++ sumi = vdotq_laneq_s32(sumi, qx[5], y.val[1], 1); ++ sumi = vdotq_laneq_s32(sumi, qx[6], y.val[1], 2); ++ sumi = vdotq_laneq_s32(sumi, qx[7], y.val[1], 3); ++ isum[iy] = vmlaq_s32(isum[iy], sumi, scales.val[l]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto d8 = vdupq_n_f32(q8.scale(iy, ibl)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(d4h, d8), vcvtq_f32_s32(isum[iy])); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix+0, iy, acc[2*iy+0]); ++ info.store(ix+4, iy, acc[2*iy+1]); ++ acc[2*iy+0] = acc[2*iy+1] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++static void mul_mat_q8_KV_q8_KV_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%32 == 0); ++ int32x4_t acc[4] = {}; ++ auto dptr = (const float *)info.src1_row(0); ++ const float dy = dptr[0]; ++ auto q8y = (const int8_t *)(dptr + 2); ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto dx = (const float *)((const char *)vx + ix*bx); ++ auto q8x = (const int8_t *)(dx + 2); ++ for (int i = 0; i < n/64; ++i) { ++ auto qx = vld1q_s8_x4(q8x + 64*i); ++ for (int j = 0; j < 4; ++j) { ++ acc[j] = ggml_vdotq_s32(acc[j], qx.val[j], vld1q_s8(q8y + 64*i + 16*j)); ++ } ++ } ++ if (int i = 2*(n/64); i < n/32) { ++ auto qx = vld1q_s8_x2(q8x + 32*i); ++ for (int j = 0; j < 2; ++j) { ++ acc[j] = ggml_vdotq_s32(acc[j], qx.val[j], vld1q_s8(q8y + 32*i + 16*j)); ++ } ++ } ++ acc[0] = vaddq_s32(acc[0], acc[1]); ++ acc[2] = vaddq_s32(acc[2], acc[3]); ++ acc[0] = vaddq_s32(acc[0], acc[2]); ++ info.store(ix, 0, dx[0]*dy*vaddvq_s32(acc[0])); ++ acc[0] = acc[1] = acc[2] = acc[3] = vdupq_n_s32(0); ++ } ++} ++ ++template ++static void mul_mat_q8_KV_q8_KV(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ GGML_ASSERT(n%16 == 0); ++ int8x16_t qx[4]; ++ int32x4_t acc[nrc_y] = {}; ++ float dy[nrc_y]; ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ const int8_t * q8x[4]; ++ float dx[4]; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ for (int kx = 0; kx < 4; ++kx) { ++ auto dptr = (const float *)((const char *)vx + (ix+kx)*bx); ++ dx[kx] = dptr[0]; ++ q8x[kx] = (const int8_t *)(dptr + 2); ++ } ++ for (int i = 0; i < n/16; ++i) { ++ for (int kx = 0; kx < 4; ++kx) qx[kx] = vld1q_s8(q8x[kx] + 16*i); ++ auto row01 = vtrnq_s32(qx[0], qx[1]); ++ auto row23 = vtrnq_s32(qx[2], qx[3]); ++ qx[0] = vtrn1q_s64(row01.val[0], row23.val[0]); ++ qx[1] = vtrn1q_s64(row01.val[1], row23.val[1]); ++ qx[2] = vtrn2q_s64(row01.val[0], row23.val[0]); ++ qx[3] = vtrn2q_s64(row01.val[1], row23.val[1]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8y[iy] + 16*i); ++ acc[iy] = vdotq_laneq_s32(acc[iy], qx[0], y, 0); ++ acc[iy] = vdotq_laneq_s32(acc[iy], qx[1], y, 1); ++ acc[iy] = vdotq_laneq_s32(acc[iy], qx[2], y, 2); ++ acc[iy] = vdotq_laneq_s32(acc[iy], qx[3], y, 3); ++ } ++ } ++ auto scales_x = vld1q_f32(dx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scale = vmulq_f32(scales_x, vdupq_n_f32(dy[iy])); ++ info.store(ix, iy, vmulq_f32(scale, vcvtq_f32_s32(acc[iy]))); ++ acc[iy] = vdupq_n_s32(0); ++ } ++ } ++} ++ ++template ++void mul_mat_q8_KV_r8_q8_KV(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ int32x4_t acc[2*nrc_y] = {}; ++ float dy[nrc_y]; ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const float * dptr = (const float *)((const char *)vx + ix*bx); ++ auto q8x = (const int8_t *)(dptr + 8); ++ for (int ib = 0; ib < n/16; ++ib) { ++ auto q1 = vld1q_s8_x4(q8x + 128*ib + 0); ++ auto q2 = vld1q_s8_x4(q8x + 128*ib + 64); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8(q8y[iy]+16*ib); ++ acc[2*iy+0] = vdotq_laneq_s32(acc[2*iy+0], q1.val[0], y, 0); ++ acc[2*iy+1] = vdotq_laneq_s32(acc[2*iy+1], q1.val[1], y, 0); ++ acc[2*iy+0] = vdotq_laneq_s32(acc[2*iy+0], q1.val[2], y, 1); ++ acc[2*iy+1] = vdotq_laneq_s32(acc[2*iy+1], q1.val[3], y, 1); ++ acc[2*iy+0] = vdotq_laneq_s32(acc[2*iy+0], q2.val[0], y, 2); ++ acc[2*iy+1] = vdotq_laneq_s32(acc[2*iy+1], q2.val[1], y, 2); ++ acc[2*iy+0] = vdotq_laneq_s32(acc[2*iy+0], q2.val[2], y, 3); ++ acc[2*iy+1] = vdotq_laneq_s32(acc[2*iy+1], q2.val[3], y, 3); ++ } ++ } ++ auto scale1_x = vld1q_f32(dptr+0); ++ auto scale2_x = vld1q_f32(dptr+4); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scale_y = vdupq_n_f32(dy[iy]); ++ auto scale1 = vmulq_f32(scale1_x, scale_y); ++ auto scale2 = vmulq_f32(scale2_x, scale_y); ++ info.store(ix+0, iy, vmulq_f32(scale1, vcvtq_f32_s32(acc[2*iy+0]))); ++ info.store(ix+4, iy, vmulq_f32(scale2, vcvtq_f32_s32(acc[2*iy+1]))); ++ acc[2*iy+0] = acc[2*iy+1] = vdupq_n_s32(0.f); ++ } ++ } ++} ++ ++typedef struct { ++ ggml_half d[16]; ++ int8_t qs[8*QK8_1]; ++} block_q8_1_r8; ++ ++void iqk_convert_q2_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q2_K * x8[8]; ++ ++ block_q8_k_r8 * y = (block_q8_k_r8 *)vy; ++ ++ float32_t f_values[QK_K]; ++ uint32_t block[8]; ++ ++ int8x16x2_t xv[4]; ++ ++ auto ml = vdupq_n_u8(0x03); ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q2_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ auto vd = vdupq_n_f32(GGML_FP16_TO_FP32(x8[k][i].d)); ++ auto vm = vdupq_n_f32(-GGML_FP16_TO_FP32(x8[k][i].dmin)); ++ auto block_max = vdupq_n_f32(0); ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto bits = vld1q_u8_x2(x8[k][i].qs+32*i128); ++ xv[0].val[0] = vandq_u8(bits.val[0], ml); ++ xv[0].val[1] = vandq_u8(bits.val[1], ml); ++ xv[1].val[0] = vandq_u8(vshrq_n_u8(bits.val[0], 2), ml); ++ xv[1].val[1] = vandq_u8(vshrq_n_u8(bits.val[1], 2), ml); ++ xv[2].val[0] = vandq_u8(vshrq_n_u8(bits.val[0], 4), ml); ++ xv[2].val[1] = vandq_u8(vshrq_n_u8(bits.val[1], 4), ml); ++ xv[3].val[0] = vshrq_n_u8(bits.val[0], 6); ++ xv[3].val[1] = vshrq_n_u8(bits.val[1], 6); ++ for (int l = 0; l < 4; ++l) { ++ auto d1 = vdupq_n_s8(x8[k][i].scales[8*i128 + 2*l + 0] & 0xf); ++ auto d2 = vdupq_n_s8(x8[k][i].scales[8*i128 + 2*l + 1] & 0xf); ++ auto q1_8 = vmulq_s8(d1, xv[l].val[0]); ++ auto q2_8 = vmulq_s8(d2, xv[l].val[1]); ++ auto q1_16_1 = vmovl_s8(vget_low_s8 (q1_8)); ++ auto q1_16_2 = vmovl_s8(vget_high_s8(q1_8)); ++ auto q2_16_1 = vmovl_s8(vget_low_s8 (q2_8)); ++ auto q2_16_2 = vmovl_s8(vget_high_s8(q2_8)); ++ float32x4x4_t f1{vcvtq_f32_s32(vmovl_s16(vget_low_s16(q1_16_1))), vcvtq_f32_s32(vmovl_s16(vget_high_s16(q1_16_1))), ++ vcvtq_f32_s32(vmovl_s16(vget_low_s16(q1_16_2))), vcvtq_f32_s32(vmovl_s16(vget_high_s16(q1_16_2)))}; ++ float32x4x4_t f2{vcvtq_f32_s32(vmovl_s16(vget_low_s16(q2_16_1))), vcvtq_f32_s32(vmovl_s16(vget_high_s16(q2_16_1))), ++ vcvtq_f32_s32(vmovl_s16(vget_low_s16(q2_16_2))), vcvtq_f32_s32(vmovl_s16(vget_high_s16(q2_16_2)))}; ++ ++ auto m1 = vmulq_f32(vm, vcvtq_f32_s32(vdupq_n_s32(x8[k][i].scales[8*i128 + 2*l + 0] >> 4))); ++ auto m2 = vmulq_f32(vm, vcvtq_f32_s32(vdupq_n_s32(x8[k][i].scales[8*i128 + 2*l + 1] >> 4))); ++ ++ for (int j = 0; j < 4; ++j) { ++ f1.val[j] = vfmaq_f32(m1, vd, f1.val[j]); ++ f2.val[j] = vfmaq_f32(m2, vd, f2.val[j]); ++ } ++ vst1q_f32_x4(f_values + 128*i128 + 32*l + 0, f1); ++ vst1q_f32_x4(f_values + 128*i128 + 32*l + 16, f2); ++ ++ auto max1 = vmaxq_f32(vmaxq_f32(vabsq_f32(f1.val[0]), vabsq_f32(f1.val[1])), vmaxq_f32(vabsq_f32(f1.val[2]), vabsq_f32(f1.val[3]))); ++ auto max2 = vmaxq_f32(vmaxq_f32(vabsq_f32(f2.val[0]), vabsq_f32(f2.val[1])), vmaxq_f32(vabsq_f32(f2.val[2]), vabsq_f32(f2.val[3]))); ++ block_max = vmaxq_f32(block_max, vmaxq_f32(max1, max2)); ++ } ++ } ++ auto max = vmaxvq_f32(block_max); ++ float d = max / 127.f; ++ auto id = vdupq_n_f32(d != 0.0f ? 1/d : 0.0f); ++ y[i].d[k] = GGML_FP32_TO_FP16(d); ++ int16x8x4_t i16; ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto v1 = vld1q_f32_x4(f_values + 32*ib32 + 0); ++ auto v2 = vld1q_f32_x4(f_values + 32*ib32 + 16); ++ i16.val[0] = vcombine_s16(vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v1.val[0]))), vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v1.val[1])))); ++ i16.val[1] = vcombine_s16(vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v1.val[2]))), vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v1.val[3])))); ++ i16.val[2] = vcombine_s16(vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v2.val[0]))), vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v2.val[1])))); ++ i16.val[3] = vcombine_s16(vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v2.val[2]))), vmovn_s32(vcvtnq_s32_f32(vmulq_f32(id, v2.val[3])))); ++ vst1q_s8((int8_t *)block + 0, vcombine_s8(vmovn_s16(i16.val[0]), vmovn_s16(i16.val[1]))); ++ vst1q_s8((int8_t *)block + 16, vcombine_s8(vmovn_s16(i16.val[2]), vmovn_s16(i16.val[3]))); ++ auto q8 = (uint32_t *)y[i].qs + 64*ib32; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++void iqk_convert_q3_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q3_K * x8[8]; ++ ++ block_q8_k_r8 * y = (block_q8_k_r8 *)vy; ++ ++ uint32_t block[8]; ++ int8x16x2_t xv[8]; ++ uint32_t aux32[4]; ++ ++ auto ml = vdupq_n_s8(0x03); ++ auto mh = vdupq_n_s8(0x04); ++ ++ union { int8x16_t vec; int8_t val[16]; } helper; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q3_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ auto sc16 = (const uint16_t *)x8[k][i].scales; ++ uint32_t aux0 = sc16[0] | (sc16[1] << 16); ++ uint32_t aux1 = sc16[2] | (sc16[3] << 16); ++ uint32_t aux2 = sc16[4] | (sc16[5] << 16); ++ aux32[0] = (aux0 & 0x0f0f0f0f) | ((aux2 << 4) & 0x30303030); ++ aux32[1] = (aux1 & 0x0f0f0f0f) | ((aux2 << 2) & 0x30303030); ++ aux32[2] = ((aux0 >> 4) & 0x0f0f0f0f) | ((aux2 >> 0) & 0x30303030); ++ aux32[3] = ((aux1 >> 4) & 0x0f0f0f0f) | ((aux2 >> 2) & 0x30303030); ++ helper.vec = vaddq_s8(vld1q_s8((const int8_t *)aux32), vdupq_n_s8(-32)); ++ auto hbits = vld1q_u8_x2(x8[k][i].hmask); ++ auto max_i16 = vdupq_n_u16(0); ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto q2bits = vld1q_u8_x2(x8[k][i].qs + 32*i128); ++ xv[4*i128+0].val[0] = vsubq_s8(vorrq_s8(vandq_s8(q2bits.val[0], ml), vandq_s8(vshlq_n_u8(hbits.val[0], 2), mh)), mh); ++ xv[4*i128+0].val[1] = vsubq_s8(vorrq_s8(vandq_s8(q2bits.val[1], ml), vandq_s8(vshlq_n_u8(hbits.val[1], 2), mh)), mh); ++ xv[4*i128+1].val[0] = vsubq_s8(vorrq_s8(vandq_s8(vshrq_n_u8(q2bits.val[0], 2), ml), vandq_s8(vshlq_n_u8(hbits.val[0], 1), mh)), mh); ++ xv[4*i128+1].val[1] = vsubq_s8(vorrq_s8(vandq_s8(vshrq_n_u8(q2bits.val[1], 2), ml), vandq_s8(vshlq_n_u8(hbits.val[1], 1), mh)), mh); ++ xv[4*i128+2].val[0] = vsubq_s8(vorrq_s8(vandq_s8(vshrq_n_u8(q2bits.val[0], 4), ml), vandq_s8(hbits.val[0], mh)), mh); ++ xv[4*i128+2].val[1] = vsubq_s8(vorrq_s8(vandq_s8(vshrq_n_u8(q2bits.val[1], 4), ml), vandq_s8(hbits.val[1], mh)), mh); ++ xv[4*i128+3].val[0] = vsubq_s8(vorrq_s8(vshrq_n_u8(q2bits.val[0], 6), vandq_s8(vshrq_n_u8(hbits.val[0], 1), mh)), mh); ++ xv[4*i128+3].val[1] = vsubq_s8(vorrq_s8(vshrq_n_u8(q2bits.val[1], 6), vandq_s8(vshrq_n_u8(hbits.val[1], 1), mh)), mh); ++ hbits.val[0] = vshrq_n_u8(hbits.val[0], 4); ++ hbits.val[1] = vshrq_n_u8(hbits.val[1], 4); ++ ++ for (int l = 0; l < 4; ++l) { ++ auto s1 = vdup_n_s8(helper.val[8*i128+2*l+0]); ++ auto s2 = vdup_n_s8(helper.val[8*i128+2*l+1]); ++ auto q16_1 = vmull_s8(s1, vget_low_s8 (xv[4*i128+l].val[0])); ++ auto q16_2 = vmull_s8(s1, vget_high_s8(xv[4*i128+l].val[0])); ++ auto q16_3 = vmull_s8(s2, vget_low_s8 (xv[4*i128+l].val[1])); ++ auto q16_4 = vmull_s8(s2, vget_high_s8(xv[4*i128+l].val[1])); ++ auto max1 = vmaxq_s16(vabsq_s16(q16_1), vabsq_s16(q16_2)); ++ auto max2 = vmaxq_s16(vabsq_s16(q16_3), vabsq_s16(q16_4)); ++ max_i16 = vmaxq_s16(max_i16, vmaxq_s16(max1, max2)); ++ } ++ } ++ auto imax16 = vmaxvq_s16(max_i16); ++ bool needs_scaling = true; ++ float dnew = float(imax16) / 127; ++ if (dnew < 1.f) { ++ dnew = 1.f; needs_scaling = false; ++ } ++ d *= dnew; ++ y[i].d[k] = GGML_FP32_TO_FP16(d); ++ auto scale = vdupq_n_f32(std::abs(dnew) > 1e-9f ? 1/dnew : 0.f); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto s1 = vdup_n_s8(helper.val[2*ib32+0]); ++ auto s2 = vdup_n_s8(helper.val[2*ib32+1]); ++ auto q16_1 = vmull_s8(s1, vget_low_s8 (xv[ib32].val[0])); ++ auto q16_2 = vmull_s8(s1, vget_high_s8(xv[ib32].val[0])); ++ auto q16_3 = vmull_s8(s2, vget_low_s8 (xv[ib32].val[1])); ++ auto q16_4 = vmull_s8(s2, vget_high_s8(xv[ib32].val[1])); ++ if (needs_scaling) { ++ int32x4x4_t i1{vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (q16_1))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(q16_1))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (q16_2))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(q16_2)))))}; ++ int32x4x4_t i2{vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (q16_3))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(q16_3))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (q16_4))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(q16_4)))))}; ++ int16x8x4_t i3{vcombine_s16(vmovn_s32(i1.val[0]), vmovn_s32(i1.val[1])), ++ vcombine_s16(vmovn_s32(i1.val[2]), vmovn_s32(i1.val[3])), ++ vcombine_s16(vmovn_s32(i2.val[0]), vmovn_s32(i2.val[1])), ++ vcombine_s16(vmovn_s32(i2.val[2]), vmovn_s32(i2.val[3]))}; ++ vst1q_s8((int8_t *)block + 0, vcombine_s8(vmovn_s16(i3.val[0]), vmovn_s16(i3.val[1]))); ++ vst1q_s8((int8_t *)block + 16, vcombine_s8(vmovn_s16(i3.val[2]), vmovn_s16(i3.val[3]))); ++ } else { ++ vst1q_s8((int8_t *)block + 0, vcombine_s8(vmovn_s16(q16_1), vmovn_s16(q16_2))); ++ vst1q_s8((int8_t *)block + 16, vcombine_s8(vmovn_s16(q16_3), vmovn_s16(q16_4))); ++ } ++ auto qs = (uint32_t *)y[i].qs + 64*ib32; ++ for (int l = 0; l < 8; ++l) { ++ qs[8*l + k] = block[l]; ++ } ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++void iqk_convert_q6_k_q8_0_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q6_K * x8[8]; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ float all_s[64]; ++ uint32_t block[8]; ++ int8x16x2_t xv[8]; ++ ++ auto ml = vdupq_n_u8(0x0f); ++ auto mh = vdupq_n_u8(0x30); ++ auto m32 = vdupq_n_s8(-32); ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q6_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ auto ql = x8[k][i].ql; ++ auto qh = x8[k][i].qh; ++ for (int i128 = 0; i128 < 2; ++i128) { ++ auto lbits1 = vld1q_u8_x2(ql + 64*i128 + 0); ++ auto lbits2 = vld1q_u8_x2(ql + 64*i128 + 32); ++ auto hbits = vld1q_u8_x2(qh + 32*i128); ++ xv[4*i128+0].val[0] = vaddq_s8(m32, vorrq_u8(vandq_u8(lbits1.val[0], ml), vandq_u8(vshlq_n_u8(hbits.val[0], 4), mh))); ++ xv[4*i128+0].val[1] = vaddq_s8(m32, vorrq_u8(vandq_u8(lbits1.val[1], ml), vandq_u8(vshlq_n_u8(hbits.val[1], 4), mh))); ++ xv[4*i128+1].val[0] = vaddq_s8(m32, vorrq_u8(vandq_u8(lbits2.val[0], ml), vandq_u8(vshlq_n_u8(hbits.val[0], 2), mh))); ++ xv[4*i128+1].val[1] = vaddq_s8(m32, vorrq_u8(vandq_u8(lbits2.val[1], ml), vandq_u8(vshlq_n_u8(hbits.val[1], 2), mh))); ++ xv[4*i128+2].val[0] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits1.val[0], 4), vandq_u8(hbits.val[0], mh))); ++ xv[4*i128+2].val[1] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits1.val[1], 4), vandq_u8(hbits.val[1], mh))); ++ xv[4*i128+3].val[0] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits2.val[0], 4), vandq_u8(vshrq_n_u8(hbits.val[0], 2), mh))); ++ xv[4*i128+3].val[1] = vaddq_s8(m32, vorrq_u8(vshrq_n_u8(lbits2.val[1], 4), vandq_u8(vshrq_n_u8(hbits.val[1], 2), mh))); ++ } ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ // We have two blocks of 16 with different scales ++ // We multiply the quants with the scales, find the max value, and convert to 8-bit quants with a single block scale. ++ auto s1 = vdup_n_s8(x8[k][i].scales[2*ib32+0]); ++ auto s2 = vdup_n_s8(x8[k][i].scales[2*ib32+1]); ++ int16x8x4_t i16{vmull_s8(s1, vget_low_s8(xv[ib32].val[0])), vmull_s8(s1, vget_high_s8(xv[ib32].val[0])), ++ vmull_s8(s2, vget_low_s8(xv[ib32].val[1])), vmull_s8(s2, vget_high_s8(xv[ib32].val[1]))}; ++ auto imax16 = vmaxq_u16(vmaxq_u16(vabsq_s16(i16.val[0]), vabsq_s16(i16.val[1])), vmaxq_u16(vabsq_s16(i16.val[2]), vabsq_s16(i16.val[3]))); ++ auto imax = vmaxvq_u16(imax16); ++ float max = float(imax) / 127; ++ all_s[8*ib32+k] = d*max; ++ if (max > 1e-9f) { ++ auto scale = vdupq_n_f32(1/max); ++ int32x4x4_t i32_1 = {vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (i16.val[0]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(i16.val[0]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (i16.val[1]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(i16.val[1])))))}; ++ int32x4x4_t i32_2 = {vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (i16.val[2]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(i16.val[2]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (i16.val[3]))))), ++ vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(i16.val[3])))))}; ++ i16.val[0] = vcombine_s16(vmovn_s32(i32_1.val[0]), vmovn_s32(i32_1.val[1])); ++ i16.val[1] = vcombine_s16(vmovn_s32(i32_1.val[2]), vmovn_s32(i32_1.val[3])); ++ i16.val[2] = vcombine_s16(vmovn_s32(i32_2.val[0]), vmovn_s32(i32_2.val[1])); ++ i16.val[3] = vcombine_s16(vmovn_s32(i32_2.val[2]), vmovn_s32(i32_2.val[3])); ++ vst1q_s8((int8_t *)block + 0, vcombine_s8(vmovn_s16(i16.val[0]), vmovn_s16(i16.val[1]))); ++ vst1q_s8((int8_t *)block + 16, vcombine_s8(vmovn_s16(i16.val[2]), vmovn_s16(i16.val[3]))); ++ } else { ++ std::memset(block, 0, 8*sizeof(uint32_t)); ++ } ++ auto qs = (uint32_t *)y[ib32].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ vst1_f16((float16_t *)y[ib32].d + 0, vcvt_f16_f32(vld1q_f32(all_s + 8*ib32 + 0))); ++ vst1_f16((float16_t *)y[ib32].d + 4, vcvt_f16_f32(vld1q_f32(all_s + 8*ib32 + 4))); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++template ++static void mul_mat_q6_k_q8_0_x4(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ DequantizerQ6K deq(vx, bx, nrc_y); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ float32x4_t acc[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ auto scales = deq.new_block(i); ++ ++ deq.prepare_signed(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][2*i+0].qs); ++ auto dot1 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[0], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[1], y.val[1])); ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+32); ++ auto dot2 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[2], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[3], y.val[1])); ++ auto dot12 = vpaddq_s32(dot1, dot2); // 0, 1, 2, 3 ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+64); ++ auto dot3 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[0], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[1], y.val[1])); ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+96); ++ auto dot4 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[2], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[3], y.val[1])); ++ auto dot34 = vpaddq_s32(dot3, dot4); // 4, 5, 6, 7 ++ auto d8 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+0].d)); ++ auto d8_1 = vzip1q_f32(d8, d8); ++ auto d8_2 = vzip2q_f32(d8, d8); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[0], d8_1), vcvtq_f32_s32(dot12)); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[1], d8_2), vcvtq_f32_s32(dot34)); ++ } ++ ++ deq.prepare_signed(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][2*i+1].qs); ++ auto dot1 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[0], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[1], y.val[1])); ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+32); ++ auto dot2 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[2], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[3], y.val[1])); ++ auto dot12 = vpaddq_s32(dot1, dot2); // 0, 1, 2, 3 ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+64); ++ auto dot3 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[0], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[1], y.val[1])); ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+96); ++ auto dot4 = vpaddq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[2], y.val[0]), ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[3], y.val[1])); ++ auto dot34 = vpaddq_s32(dot3, dot4); // 4, 5, 6, 7 ++ auto d8 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+1].d)); ++ auto d8_1 = vzip1q_f32(d8, d8); ++ auto d8_2 = vzip2q_f32(d8, d8); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[2], d8_1), vcvtq_f32_s32(dot12)); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[3], d8_2), vcvtq_f32_s32(dot34)); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, vaddvq_f32(acc[iy])); ++ } ++ } ++} ++ ++void iqk_convert_q4_k_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q4_K * x8[8]; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ ggml_half dh[16]; ++ uint16_t all_ls[128]; ++ ++ uint32_t utmp[4]; ++ const uint8_t * u8 = (const uint8_t *)utmp; ++ uint32_t block[8]; ++ ++ auto ml = vdupq_n_u8(0xf); ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q4_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ dh[k+0] = x8[k][i].d; ++ dh[k+8] = x8[k][i].dmin; ++ make_q4_scales(x8[k][i].scales, utmp); ++ auto qs = x8[k][i].qs; ++ for (int ib64 = 0; ib64 < 4; ++ib64) { ++ all_ls[8*(2*ib64 + 0) + k ] = u8[2*ib64+0]; ++ all_ls[8*(2*ib64 + 1) + k ] = u8[2*ib64+1]; ++ all_ls[8*(2*ib64 + 0) + k + 64] = u8[2*ib64+8]; ++ all_ls[8*(2*ib64 + 1) + k + 64] = u8[2*ib64+9]; ++ auto bits = vld1q_u8_x2(qs+32*ib64); ++ uint8x16x2_t xv1{vandq_u8(bits.val[0], ml), vandq_u8(bits.val[1], ml)}; ++ uint8x16x2_t xv2{vshrq_n_u8(bits.val[0], 4), vshrq_n_u8(bits.val[1], 4)}; ++ vst1q_u8_x2((uint8_t *)block, xv1); ++ auto q8 = (uint32_t *)y[2*ib64+0].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ vst1q_u8_x2((uint8_t *)block, xv2); ++ q8 = (uint32_t *)y[2*ib64+1].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ float32x4x2_t vd{ vcvt_f32_f16(vld1_f16((const float16_t *)dh+0)), vcvt_f32_f16(vld1_f16((const float16_t *)dh+ 4)) }; ++ float32x4x2_t vm{ vcvt_f32_f16(vld1_f16((const float16_t *)dh+8)), vcvt_f32_f16(vld1_f16((const float16_t *)dh+12)) }; ++ vm.val[0] = vmulq_f32(vdupq_n_f32(-1.f), vm.val[0]); ++ vm.val[1] = vmulq_f32(vdupq_n_f32(-1.f), vm.val[1]); ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ auto iscales16 = vld1q_u16(all_ls + 8*ib32); ++ uint32x4x2_t iscales32 = { vmovl_u16(vget_low_u16(iscales16)), vmovl_u16(vget_high_u16(iscales16)) }; ++ auto scales1 = vmulq_f32(vd.val[0], vcvtq_f32_u32(iscales32.val[0])); ++ auto scales2 = vmulq_f32(vd.val[1], vcvtq_f32_u32(iscales32.val[1])); ++ vst1_f16((float16_t *)y[ib32].d+0, vcvt_f16_f32(scales1)); ++ vst1_f16((float16_t *)y[ib32].d+4, vcvt_f16_f32(scales2)); ++ ++ iscales16 = vld1q_u16(all_ls + 8*ib32 + 64); ++ iscales32 = { vmovl_u16(vget_low_u16(iscales16)), vmovl_u16(vget_high_u16(iscales16)) }; ++ scales1 = vmulq_f32(vm.val[0], vcvtq_f32_u32(iscales32.val[0])); ++ scales2 = vmulq_f32(vm.val[1], vcvtq_f32_u32(iscales32.val[1])); ++ vst1_f16((float16_t *)y[ib32].d+ 8, vcvt_f16_f32(scales1)); ++ vst1_f16((float16_t *)y[ib32].d+12, vcvt_f16_f32(scales2)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++void iqk_convert_q5_k_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_q5_K * x8[8]; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ ggml_half dh[16]; ++ uint16_t all_ls[128]; ++ ++ uint32_t utmp[4]; ++ const uint8_t * u8 = (const uint8_t *)utmp; ++ uint32_t block[8]; ++ ++ auto ml = vdupq_n_u8(0x0f); ++ auto mh = vdupq_n_u8(0x10); ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q5_K *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ dh[k+0] = x8[k][i].d; ++ dh[k+8] = x8[k][i].dmin; ++ make_q4_scales(x8[k][i].scales, utmp); ++ auto qs = x8[k][i].qs; ++ auto hbits = vld1q_u8_x2(x8[k][i].qh); ++ for (int ib64 = 0; ib64 < 4; ++ib64) { ++ all_ls[8*(2*ib64 + 0) + k ] = u8[2*ib64+0]; ++ all_ls[8*(2*ib64 + 1) + k ] = u8[2*ib64+1]; ++ all_ls[8*(2*ib64 + 0) + k + 64] = u8[2*ib64+8]; ++ all_ls[8*(2*ib64 + 1) + k + 64] = u8[2*ib64+9]; ++ auto bits = vld1q_u8_x2(qs+32*ib64); ++ uint8x16x2_t xv1{vandq_u8(bits.val[0], ml), vandq_u8(bits.val[1], ml)}; ++ uint8x16x2_t xv2{vshrq_n_u8(bits.val[0], 4), vshrq_n_u8(bits.val[1], 4)}; ++ xv1.val[0] = vorrq_u8(xv1.val[0], vandq_u8(vshlq_n_u8(hbits.val[0], 4), mh)); ++ xv1.val[1] = vorrq_u8(xv1.val[1], vandq_u8(vshlq_n_u8(hbits.val[1], 4), mh)); ++ xv2.val[0] = vorrq_u8(xv2.val[0], vandq_u8(vshlq_n_u8(hbits.val[0], 3), mh)); ++ xv2.val[1] = vorrq_u8(xv2.val[1], vandq_u8(vshlq_n_u8(hbits.val[1], 3), mh)); ++ vst1q_u8_x2((uint8_t *)block, xv1); ++ auto q8 = (uint32_t *)y[2*ib64+0].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ vst1q_u8_x2((uint8_t *)block, xv2); ++ q8 = (uint32_t *)y[2*ib64+1].qs; ++ for (int l = 0; l < 4; ++l) { ++ q8[8*l + k + 0] = block[l + 0]; ++ q8[8*l + k + 32] = block[l + 4]; ++ } ++ hbits.val[0] = vshrq_n_u8(hbits.val[0], 2); ++ hbits.val[1] = vshrq_n_u8(hbits.val[1], 2); ++ } ++ } ++ float32x4x2_t vd{ vcvt_f32_f16(vld1_f16((const float16_t *)dh+0)), vcvt_f32_f16(vld1_f16((const float16_t *)dh+ 4)) }; ++ float32x4x2_t vm{ vcvt_f32_f16(vld1_f16((const float16_t *)dh+8)), vcvt_f32_f16(vld1_f16((const float16_t *)dh+12)) }; ++ vm.val[0] = vmulq_f32(vdupq_n_f32(-1.f), vm.val[0]); ++ vm.val[1] = vmulq_f32(vdupq_n_f32(-1.f), vm.val[1]); ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ auto iscales16 = vld1q_u16(all_ls + 8*ib32); ++ uint32x4x2_t iscales32 = { vmovl_u16(vget_low_u16(iscales16)), vmovl_u16(vget_high_u16(iscales16)) }; ++ auto scales1 = vmulq_f32(vd.val[0], vcvtq_f32_u32(iscales32.val[0])); ++ auto scales2 = vmulq_f32(vd.val[1], vcvtq_f32_u32(iscales32.val[1])); ++ vst1_f16((float16_t *)y[ib32].d+0, vcvt_f16_f32(scales1)); ++ vst1_f16((float16_t *)y[ib32].d+4, vcvt_f16_f32(scales2)); ++ ++ iscales16 = vld1q_u16(all_ls + 8*ib32 + 64); ++ iscales32 = { vmovl_u16(vget_low_u16(iscales16)), vmovl_u16(vget_high_u16(iscales16)) }; ++ scales1 = vmulq_f32(vm.val[0], vcvtq_f32_u32(iscales32.val[0])); ++ scales2 = vmulq_f32(vm.val[1], vcvtq_f32_u32(iscales32.val[1])); ++ vst1_f16((float16_t *)y[ib32].d+ 8, vcvt_f16_f32(scales1)); ++ vst1_f16((float16_t *)y[ib32].d+12, vcvt_f16_f32(scales2)); ++ } ++ y += QK_K/32; ++ } ++ } ++} ++ ++template ++static void mul_mat_qX_k_q8_1_x4(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n % QK_K == 0); ++ const int nb = n / QK_K; ++ ++ Q8 q8(info); ++ ++ Dequantizer deq(vx, bx, nrc_y); ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ float32x4_t acc[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb; ++i) { ++ ++ auto scales = deq.new_block(i); ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto m1 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+0].d+4)); ++ auto m2 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+1].d+4)); ++ acc[iy] = vfmaq_f32(acc[iy], scales.val[2], m1); ++ acc[iy] = vfmaq_f32(acc[iy], scales.val[3], m2); ++ } ++ ++ deq.prepare(i, 0); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][2*i+0].qs); ++ auto dot1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[0], y.val[0]), deq.bits.b1.val[1], y.val[1]); ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+32); ++ auto dot2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[2], y.val[0]), deq.bits.b1.val[3], y.val[1]); ++ auto dot12 = vpaddq_s32(dot1, dot2); // 0, 0, 1, 1 ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+64); ++ auto dot3 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[0], y.val[0]), deq.bits.b2.val[1], y.val[1]); ++ y = vld1q_s8_x2(q8.y[iy][2*i+0].qs+96); ++ auto dot4 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[2], y.val[0]), deq.bits.b2.val[3], y.val[1]); ++ auto dot34 = vpaddq_s32(dot3, dot4); // 2, 2, 3, 3 ++ auto dot = vpaddq_s32(dot12, dot34); // 0, 1, 2, 3 ++ auto d8 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+0].d)); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[0], d8), vcvtq_f32_s32(dot)); ++ } ++ ++ deq.prepare(i, 1); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][2*i+1].qs); ++ auto dot1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[0], y.val[0]), deq.bits.b1.val[1], y.val[1]); ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+32); ++ auto dot2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b1.val[2], y.val[0]), deq.bits.b1.val[3], y.val[1]); ++ auto dot12 = vpaddq_s32(dot1, dot2); // 0, 0, 1, 1 ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+64); ++ auto dot3 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[0], y.val[0]), deq.bits.b2.val[1], y.val[1]); ++ y = vld1q_s8_x2(q8.y[iy][2*i+1].qs+96); ++ auto dot4 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b2.val[2], y.val[0]), deq.bits.b2.val[3], y.val[1]); ++ auto dot34 = vpaddq_s32(dot3, dot4); // 2, 2, 3, 3 ++ auto dot = vpaddq_s32(dot12, dot34); // 0, 1, 2, 3 ++ auto d8 = vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][2*i+1].d)); ++ acc[iy] = vfmaq_f32(acc[iy], vmulq_f32(scales.val[1], d8), vcvtq_f32_s32(dot)); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, vaddvq_f32(acc[iy])); ++ } ++ } ++} ++ ++inline float convert_to_q8_k_r8(float d0, const int8x16x2_t * qx, const int8_t * scales, uint32_t * block, uint32_t * q8_k) { ++ auto max_i16 = vdupq_n_u16(0); ++ int16x8x4_t q[8]; ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ auto scale_l = vdup_n_s8(scales[2*ib32+0]); ++ auto scale_h = vdup_n_s8(scales[2*ib32+1]); ++ q[ib32].val[0] = vmull_s8(scale_l, vget_low_s8 (qx[ib32].val[0])); ++ q[ib32].val[1] = vmull_s8(scale_l, vget_high_s8(qx[ib32].val[0])); ++ q[ib32].val[2] = vmull_s8(scale_h, vget_low_s8 (qx[ib32].val[1])); ++ q[ib32].val[3] = vmull_s8(scale_h, vget_high_s8(qx[ib32].val[1])); ++ max_i16 = vmaxq_u16(max_i16, vmaxq_u16(vabsq_s16(q[ib32].val[0]), vabsq_s16(q[ib32].val[1]))); ++ max_i16 = vmaxq_u16(max_i16, vmaxq_u16(vabsq_s16(q[ib32].val[2]), vabsq_s16(q[ib32].val[3]))); ++ } ++ uint16_t imax = vmaxvq_u16(max_i16); ++ if (!imax) { ++ for (int ib32 = 0; ib32 < 8; ++ib32) for (int l = 0; l < 8; ++l) q8_k[64*ib32 + 8*l] = 0; ++ return 0.f; ++ } ++ float dnew = float(imax) * d0; ++ //auto max_u32 = vmaxq_u32(vmovl_u16(vget_low_u16(max_i16)), vmovl_u16(vget_high_u16(max_i16))); ++ //auto max_f32 = vcvtq_f32_u32(max_u32); ++ //auto dnew = vmaxvq_f32(max_f32) * d0; ++ bool needs_scaling = true; ++ if (dnew <= 1.f) { ++ dnew = 1.f; needs_scaling = false; ++ } ++ auto scale = vdupq_n_f32(1/dnew); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ if (needs_scaling) { ++ for (int l = 0; l < 4; ++l) { ++ auto i1 = vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_low_s16 (q[ib32].val[l]))))); ++ auto i2 = vcvtnq_s32_f32(vmulq_f32(scale, vcvtq_f32_s32(vmovl_s16(vget_high_s16(q[ib32].val[l]))))); ++ q[ib32].val[l] = vcombine_s16(vmovn_s32(i1), vmovn_s32(i2)); ++ } ++ } ++ for (int l = 0; l < 2; ++l) { ++ auto s8 = vcombine_s8(vmovn_s16(q[ib32].val[2*l+0]), vmovn_s16(q[ib32].val[2*l+1])); ++ vst1q_s8((int8_t *)block + 16*l, s8); ++ } ++ auto qb = q8_k + 64*ib32; ++ for (int l = 0; l < 8; ++l) { ++ qb[8*l] = block[l]; ++ } ++ } ++ return dnew; ++} ++ ++// TODO: move this to iqk_gemm_iquants ++void iqk_convert_iq4_xs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK_K; ++ ++ const block_iq4_xs * x8[8]; ++ ++ block_q8_k_r8 * y = (block_q8_k_r8 *)vy; ++ ++ auto values = vld1q_s8(iq4k_values); ++ ++ int8_t ls[16]; ++ float dnew[8]; ++ uint32_t block[8]; ++ int8x16x2_t xv[8]; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_iq4_xs *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ float d = GGML_FP16_TO_FP32(x8[k][i].d); ++ for (int ib32 = 0; ib32 < 8; ++ib32) { ++ ls[2*ib32+0] = ls[2*ib32+1] = (((x8[k][i].scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((x8[k][i].scales_h >> 2*ib32) & 3) << 4)) - 32; ++ auto bits = vld1q_u8(x8[k][i].qs + 16*ib32); ++ xv[ib32].val[0] = vqtbl1q_s8(values, vandq_u8(bits, vdupq_n_u8(0xf))); ++ xv[ib32].val[1] = vqtbl1q_s8(values, vshrq_n_u8(bits, 4)); ++ } ++ dnew[k] = d * convert_to_q8_k_r8(1.f/127, xv, ls, block, (uint32_t *)y[i].qs + k); ++ } ++ vst1_f16((float16_t *)y[i].d + 0, vcvt_f16_f32(vld1q_f32(dnew+0))); ++ vst1_f16((float16_t *)y[i].d + 4, vcvt_f16_f32(vld1q_f32(dnew+4))); ++ } ++ y += nb; ++ } ++} ++ ++} ++ ++bool iqk_convert_kquants_q8X_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ switch (ggml_type(type)) { ++ case GGML_TYPE_Q2_K: iqk_convert_q2_k_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q3_K: iqk_convert_q3_k_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q4_K: iqk_convert_q4_k_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_K: iqk_convert_q5_k_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q6_K: iqk_convert_q6_k_q8_0_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_IQ4_XS: iqk_convert_iq4_xs_q8_k_r8(n, vx, bx, vy, nrc_x); break; ++ default: return false; ++ } ++ return true; ++} ++ ++bool iqk_set_kernels_kquants(int ne00, int typeA, int typeB, std::array& kernels, [[maybe_unused]] mul_mat_t& func16) { ++ ++ auto etypeA = ggml_type(typeA); ++ auto expected_type_B = etypeA == GGML_TYPE_IQ4_XS_R8 || etypeA == GGML_TYPE_Q4_K_R4 || etypeA == GGML_TYPE_Q5_K_R4 ? GGML_TYPE_Q8_K32 ++ //: etypeA == GGML_TYPE_Q8_K_R8 ? GGML_TYPE_Q8_KR8 ++ : etypeA == GGML_TYPE_Q8_KV || etypeA == GGML_TYPE_Q8_KV_R8 ? GGML_TYPE_Q8_KV ++ : etypeA == GGML_TYPE_Q6_K ? GGML_TYPE_Q8_0_X4 ++ : etypeA == GGML_TYPE_Q4_K || etypeA == GGML_TYPE_Q5_K ? GGML_TYPE_Q8_1_X4 ++ : GGML_TYPE_Q8_K; ++ ++ if (ne00%QK_K != 0 || ggml_type(typeB) != expected_type_B) { ++ return false; ++ } ++ ++ func16 = nullptr; ++ ++ switch (typeA) { ++ case GGML_TYPE_Q2_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ2K, kernels) ++ break; ++ case GGML_TYPE_Q3_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerQ3K, kernels) ++ break; ++ case GGML_TYPE_Q4_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_k_q8_1_x4, DequantizerQ4K, kernels) ++ break; ++ case GGML_TYPE_Q5_K: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_k_q8_1_x4, DequantizerQ5K, kernels) ++ break; ++ case GGML_TYPE_Q6_K: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q6_k_q8_0_x4, kernels) ++ break; ++ case GGML_TYPE_IQ4_XS: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_K_q8_K_T, DequantizerIQ4XS, kernels) ++ break; ++ case GGML_TYPE_Q2_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q2_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q3_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q3_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q4_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q4_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q5_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q5_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q6_K_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q6_k_r4_q8_k, kernels) ++ break; ++ case GGML_TYPE_IQ4_XS_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_iq4_xs_r8_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q8_K_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_k_r8_q8_k, kernels) ++ break; ++ case GGML_TYPE_Q8_KV: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_KV_q8_KV, kernels) ++ kernels[0] = mul_mat_q8_KV_q8_KV_1; ++ func16 = mul_mat_q8_KV_q8_KV<16>; ++ break; ++ case GGML_TYPE_Q8_KV_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_KV_r8_q8_KV, kernels); ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++ ++} ++ ++#endif ++ ++namespace { ++ ++#ifdef __AVX2__ ++template ++static void mul_mat_q8_KV_q8_KV_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(n%32 == 0); ++ if (nrc_y == 1 && nrc_x == 1) { ++ auto dx = (const float *)vx; ++ auto dy = (const float *)info.src1_row(0); ++#ifdef HAVE_FANCY_SIMD ++ auto sy = (const int32_t *)(dy + 1); ++ auto x = (const int8_t *)(dx + 2); ++ auto y = (const int8_t *)(dy + 2); ++ auto isum = _mm512_setzero_si512(); ++ for (int i = 0; i < n/64; ++i) { ++ auto qx = _mm512_loadu_si512((const __m512i *)x + i); ++ auto qy = _mm512_loadu_si512((const __m512i *)y + i); ++ isum = _mm512_dpbusd_epi32(isum, _mm512_add_epi8(qx, _mm512_set1_epi8(127)), qy); ++ } ++ auto isum256 = _mm256_add_epi32(_mm512_castsi512_si256(isum), _mm512_extracti32x8_epi32(isum, 1)); ++ for (int i = 2*(n/64); i < n/32; ++i) { ++ auto qx = _mm256_loadu_si256((const __m256i *)x + i); ++ auto qy = _mm256_loadu_si256((const __m256i *)y + i); ++ isum256 = _mm256_dpbusd_epi32(isum256, _mm256_add_epi8(qx, _mm256_set1_epi8(127)), qy); ++ } ++ info.store(0, 0, dx[0]*dy[0]*(hsum_i32_8(isum256) - 127*sy[0])); ++#else ++ auto x = (const int8_t *)(dx + 2); ++ auto y = (const int8_t *)(dy + 2); ++ auto isum = _mm256_setzero_si256(); ++ for (int i = 0; i < n/32; ++i) { ++ auto qx = _mm256_loadu_si256((const __m256i *)x + i); ++ auto qy = _mm256_loadu_si256((const __m256i *)y + i); ++ auto dot = _mm256_maddubs_epi16(_mm256_sign_epi8(qx, qx), _mm256_sign_epi8(qy, qx)); ++ isum = _mm256_add_epi32(isum, _mm256_madd_epi16(_mm256_set1_epi16(1), dot)); ++ } ++ info.store(0, 0, dx[0]*dy[0]*hsum_i32_8(isum)); ++#endif ++ return; ++ } ++ __m256i qx[2]; ++ __m256i acc[2*nrc_y] = {}; ++ float dy[nrc_y]; ++#ifdef HAVE_FANCY_SIMD ++ int32_t sy[nrc_y]; ++#else ++ __m256i sx[2]; ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++#ifdef HAVE_FANCY_SIMD ++ auto iptr = (const int32_t *)(dptr+1); ++ sy[iy] = -127*iptr[0]; ++#endif ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ auto dx = (const float *)((const char *)vx + ix*bx); ++ auto q8x = (const int8_t *)(dx + 2); ++ for (int i = 0; i < n/64; ++i) { ++ for (int j = 0; j < 2; ++j) { ++#ifdef HAVE_FANCY_SIMD ++ qx[j] = _mm256_add_epi8(_mm256_loadu_si256((const __m256i *)q8x + 2*i + j), _mm256_set1_epi8(127)); ++#else ++ qx[j] = _mm256_loadu_si256((const __m256i *)q8x + 2*i + j); ++ sx[j] = _mm256_sign_epi8(qx[j], qx[j]); ++#endif ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ for (int j = 0; j < 2; ++j) { ++#ifdef HAVE_FANCY_SIMD ++ acc[2*iy+j] = _mm256_dpbusd_epi32(acc[2*iy+j], qx[j], _mm256_loadu_si256((const __m256i *)q8y[iy] + 2*i + j)); ++#else ++ auto dot = _mm256_maddubs_epi16(sx[j], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i *)q8y[iy] + 2*i + j), qx[j])); ++ acc[2*iy+j] = _mm256_add_epi32(acc[2*iy+j], _mm256_madd_epi16(m1, dot)); ++#endif ++ } ++ } ++ } ++ if (int i = 2*(n/64); i < n/32) { ++#ifdef HAVE_FANCY_SIMD ++ qx[0] = _mm256_add_epi8(_mm256_loadu_si256((const __m256i *)q8x + i), _mm256_set1_epi8(127)); ++#else ++ qx[0] = _mm256_loadu_si256((const __m256i *)q8x + i); ++ sx[0] = _mm256_sign_epi8(qx[0], qx[0]); ++#endif ++ for (int iy = 0; iy < nrc_y; ++iy) { ++#ifdef HAVE_FANCY_SIMD ++ acc[2*iy] = _mm256_dpbusd_epi32(acc[2*iy], qx[0], _mm256_loadu_si256((const __m256i *)q8y[iy] + i)); ++#else ++ auto dot = _mm256_maddubs_epi16(sx[0], _mm256_sign_epi8(_mm256_loadu_si256((const __m256i *)q8y[iy] + i), qx[0])); ++ acc[2*iy] = _mm256_add_epi32(acc[2*iy], _mm256_madd_epi16(m1, dot)); ++#endif ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = hsum_i32_8(_mm256_add_epi32(acc[2*iy], acc[2*iy+1])); ++#ifdef HAVE_FANCY_SIMD ++ info.store(ix, iy, dx[0]*dy[iy]*(sumi+sy[iy])); ++#else ++ info.store(ix, iy, dx[0]*dy[iy]*sumi); ++#endif ++ acc[2*iy] = acc[2*iy+1] = _mm256_setzero_si256(); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q8_KV_q8_KV_8(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ GGML_ASSERT(n%32 == 0); ++ __m512i qx[4]; ++ __m512i acc[nrc_y <= 4 ? 2*nrc_y : nrc_y] = {}; ++ float dy[nrc_y]; ++ int32_t sy[nrc_y]; ++ const int8_t * q8y[nrc_y]; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto dptr = (const float *)info.src1_row(iy); ++ dy[iy] = dptr[0]; ++ auto iptr = (const int32_t *)(dptr + 1); ++ sy[iy] = -64*iptr[0]; ++ q8y[iy] = (const int8_t *)(dptr + 2); ++ } ++ const int8_t * q8x[8]; ++ float dx[8]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int kx = 0; kx < 8; ++kx) { ++ auto dptr = (const float *)((const char *)vx + (ix+kx)*bx); ++ dx[kx] = dptr[0]; ++ q8x[kx] = (const int8_t *)(dptr + 2); ++ } ++ for (int i = 0; i < n/32; ++i) { ++ for (int kx = 0; kx < 4; ++kx) { ++ qx[kx] = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)q8x[kx+0] + i)), ++ _mm256_loadu_si256((const __m256i *)q8x[kx+4] + i), 1); ++ } ++ auto t0 = _mm512_unpacklo_epi32(qx[0], qx[1]); ++ auto t1 = _mm512_unpacklo_epi32(qx[2], qx[3]); ++ auto t2 = _mm512_unpackhi_epi32(qx[0], qx[1]); ++ auto t3 = _mm512_unpackhi_epi32(qx[2], qx[3]); ++ qx[0] = _mm512_xor_si512(_mm512_unpacklo_epi64(t0, t1), _mm512_set1_epi8(-128)); ++ qx[1] = _mm512_xor_si512(_mm512_unpackhi_epi64(t0, t1), _mm512_set1_epi8(-128)); ++ qx[2] = _mm512_xor_si512(_mm512_unpacklo_epi64(t2, t3), _mm512_set1_epi8(-128)); ++ qx[3] = _mm512_xor_si512(_mm512_unpackhi_epi64(t2, t3), _mm512_set1_epi8(-128)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y256 = _mm256_loadu_si256((const __m256i *)q8y[iy] + i); ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); ++ if constexpr (nrc_y <= 4) { ++ acc[2*iy+0] = _mm512_dpbusd_epi32(acc[2*iy+0], qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ acc[2*iy+1] = _mm512_dpbusd_epi32(acc[2*iy+1], qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ acc[2*iy+0] = _mm512_dpbusd_epi32(acc[2*iy+0], qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ acc[2*iy+1] = _mm512_dpbusd_epi32(acc[2*iy+1], qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ } else { ++ acc[iy] = _mm512_dpbusd_epi32(acc[iy], qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ acc[iy] = _mm512_dpbusd_epi32(acc[iy], qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ acc[iy] = _mm512_dpbusd_epi32(acc[iy], qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ acc[iy] = _mm512_dpbusd_epi32(acc[iy], qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ } ++ } ++ } ++ auto scales_x = _mm256_loadu_ps(dx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ if constexpr (nrc_y <= 4) { ++ auto ss = _mm512_add_epi32(_mm512_add_epi32(acc[2*iy+0], acc[2*iy+1]), _mm512_set1_epi32(sy[iy])); ++ auto sum1 = _mm_add_epi32(_mm512_extracti32x4_epi32(ss, 0), _mm512_extracti32x4_epi32(ss, 1)); ++ auto sum2 = _mm_add_epi32(_mm512_extracti32x4_epi32(ss, 2), _mm512_extracti32x4_epi32(ss, 3)); ++ auto scale = _mm256_mul_ps(scales_x, _mm256_set1_ps(dy[iy])); ++ info.store(ix+0, iy, _mm_mul_ps(_mm256_castps256_ps128(scale), _mm_cvtepi32_ps(sum1))); ++ info.store(ix+4, iy, _mm_mul_ps(_mm256_extractf128_ps(scale, 1), _mm_cvtepi32_ps(sum2))); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_si512(); ++ } else { ++ acc[iy] = _mm512_add_epi32(acc[iy], _mm512_set1_epi32(sy[iy])); ++ auto sum1 = _mm_add_epi32(_mm512_extracti32x4_epi32(acc[iy], 0), _mm512_extracti32x4_epi32(acc[iy], 1)); ++ auto sum2 = _mm_add_epi32(_mm512_extracti32x4_epi32(acc[iy], 2), _mm512_extracti32x4_epi32(acc[iy], 3)); ++ auto scale = _mm256_mul_ps(scales_x, _mm256_set1_ps(dy[iy])); ++ info.store(ix+0, iy, _mm_mul_ps(_mm256_castps256_ps128(scale), _mm_cvtepi32_ps(sum1))); ++ info.store(ix+4, iy, _mm_mul_ps(_mm256_extractf128_ps(scale, 1), _mm_cvtepi32_ps(sum2))); ++ acc[iy] = _mm512_setzero_si512(); ++ } ++ } ++ } ++} ++#endif ++#endif ++ ++template ++inline std::pair mul_mat_kernel([[maybe_unused]] int D, int int_typeA, int nq) { ++ auto typeA = ggml_type(int_typeA); ++ constexpr int kMaxQ = 8; ++#define MAKE_FUNCS(mul_mat, n) \ ++ if (n >= kMaxQ) return std::make_pair(mul_mat, kMaxQ>, kMaxQ);\ ++ else {\ ++ switch (n) {\ ++ case 1: return std::make_pair(mul_mat, 1>, 1);\ ++ case 2: return std::make_pair(mul_mat, 2>, 2);\ ++ case 3: return std::make_pair(mul_mat, 3>, 3);\ ++ case 4: return std::make_pair(mul_mat, 4>, 4);\ ++ case 5: return std::make_pair(mul_mat, 5>, 5);\ ++ case 6: return std::make_pair(mul_mat, 6>, 6);\ ++ case 7: return std::make_pair(mul_mat, 7>, 7);\ ++ }\ ++ } ++#define MAKE_FUNCS_ONLY_NRC(mul_mat, n) \ ++ if (n >= kMaxQ) return std::make_pair(mul_mat, kMaxQ);\ ++ else {\ ++ switch (n) {\ ++ case 1: return std::make_pair(mul_mat<1>, 1);\ ++ case 2: return std::make_pair(mul_mat<2>, 2);\ ++ case 3: return std::make_pair(mul_mat<3>, 3);\ ++ case 4: return std::make_pair(mul_mat<4>, 4);\ ++ case 5: return std::make_pair(mul_mat<5>, 5);\ ++ case 6: return std::make_pair(mul_mat<6>, 6);\ ++ case 7: return std::make_pair(mul_mat<7>, 7);\ ++ }\ ++ } ++ if (typeA == GGML_TYPE_Q8_KV) { ++#ifdef __aarch64__ ++ if (nq%16 == 0) return std::make_pair(mul_mat_q8_KV_q8_KV<16>, 16); ++ if (nq == 1) return std::make_pair(mul_mat_q8_KV_q8_KV_1, 1); ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_KV_q8_KV, nq); ++#else ++ if (nq == 1) return std::make_pair(mul_mat_q8_KV_q8_KV_1<1>, 1); ++#ifdef HAVE_FANCY_SIMD ++ if (D%32 == 0 && k_step%8 == 0) { ++ if (nq%16 == 0) return std::make_pair(mul_mat_q8_KV_q8_KV_8<16>, 16); ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_KV_q8_KV_8, nq); ++ } else { ++ if (nq%16 == 0) return std::make_pair(mul_mat_q8_KV_q8_KV<16>, 16); ++ } ++#endif ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_KV_q8_KV, nq); ++#endif ++ } ++ else if (typeA == GGML_TYPE_Q8_KV_R8) { ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_KV_r8_q8_KV, nq); ++ } ++ GGML_ABORT("Fatal error"); ++} ++ ++inline std::pair mul_mat_kernel(int D, int int_typeA, int nq, int k_step) { ++ switch (k_step) { ++ case 32: return mul_mat_kernel< 32>(D, int_typeA, nq); ++ case 64: return mul_mat_kernel< 64>(D, int_typeA, nq); ++ case 128: return mul_mat_kernel<128>(D, int_typeA, nq); ++ default: GGML_ABORT("Fatal error"); ++ } ++} ++ ++} ++ ++void iqk_gemm_q8kv_fa(int D, int nq, int type_k, const char * k, size_t stride_k, DataInfo& info, int k_step) { ++ auto [mul_mat, nrc_q] = mul_mat_kernel(D, type_k, nq, k_step); ++ for (int iq = 0; iq < nq/nrc_q; ++iq) { ++ mul_mat(D, k, stride_k, info, k_step); ++ info.cur_y += nrc_q; ++ } ++ int iq = nrc_q*(nq/nrc_q); ++ if (iq < nq) { ++ auto [mul_mat1, nrc_q1] = mul_mat_kernel(D, type_k, nq - iq, k_step); ++ GGML_ASSERT(nrc_q1 == nq - iq); ++ mul_mat1(D, k, stride_k, info, k_step); ++ } ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.h b/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_kquants.h +@@ -0,0 +1,15 @@ ++#pragma once ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#include ++ ++bool iqk_set_kernels_kquants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16); ++ ++void iqk_gemm_q8kv_fa(int D, int nq, int type_k, const char * k, size_t stride_k, DataInfo& info, int k_step); ++ ++bool iqk_convert_kquants_q8X_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x); ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_ktquants.h b/llama.cpp/ggml/src/iqk/iqk_gemm_ktquants.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_ktquants.h +@@ -0,0 +1,13 @@ ++#pragma once ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#include ++ ++bool iqk_set_kernels_ktquants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16); ++ ++bool iqk_dequantize_ktquants(int type, int n, const void * vx, size_t bx, void * vy, size_t stride_y, int nrc_x); ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.cpp b/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.cpp +@@ -0,0 +1,3521 @@ ++#include "iqk_gemm_legacy_quants.h" ++ ++#include ++ ++#ifdef IQK_IMPLEMENT ++ ++#include "ggml-impl.h" ++#include "iqk_utils.h" ++ ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++ ++// ++// ============================== Legacy quants ++// ++ ++#ifdef __x86_64__ ++ ++namespace { ++ ++struct DotHelper { ++ const __m256i m1 = _mm256_set1_epi16(1); ++#if defined(__AVX512VNNI__) && defined(__AVX512VL__) ++ inline __m256i dot(__m256i x, __m256i y) const { ++ return _mm256_dpbusd_epi32(_mm256_setzero_si256(), x, y); ++ } ++#else ++ inline __m256i dot(__m256i x, __m256i y) const { ++ return _mm256_madd_epi16(m1, _mm256_maddubs_epi16(x, y)); ++ } ++#endif ++}; ++ ++struct SignedDot { ++ DotHelper helper; ++ inline __m256i compute(__m256i x, __m256i y) const { ++ return helper.dot(_mm256_sign_epi8(x, x), _mm256_sign_epi8(y, x)); ++ } ++}; ++struct UnsignedDot { ++ DotHelper helper; ++ inline __m256i compute(__m256i x, __m256i y) const { ++ return helper.dot(x, y); ++ } ++}; ++ ++template struct Sum4 { ++ Dot dot; ++ inline __m256i compute(const __m256i * qx, const Q8 * y) const { ++ const Q8x4 * y4 = (const Q8x4 *)y; ++ const __m256i p0 = dot.compute(qx[0], _mm256_loadu_si256((const __m256i *)y4->qs+0)); // 8x block 0 ++ const __m256i p1 = dot.compute(qx[1], _mm256_loadu_si256((const __m256i *)y4->qs+1)); // 8x block 1 ++ const __m256i p2 = dot.compute(qx[2], _mm256_loadu_si256((const __m256i *)y4->qs+2)); // 8x block 2 ++ const __m256i p3 = dot.compute(qx[3], _mm256_loadu_si256((const __m256i *)y4->qs+3)); // 8x block 3 ++ if constexpr (can_pack) { ++ const __m256i p01 = _mm256_madd_epi16(dot.helper.m1, _mm256_packs_epi32(p0, p1)); // 0,0, 1,1, 0,0, 1,1 ++ const __m256i p23 = _mm256_madd_epi16(dot.helper.m1, _mm256_packs_epi32(p2, p3)); // 2,2, 3,3, 2,2, 3,3 ++ return _mm256_madd_epi16(dot.helper.m1, _mm256_packs_epi32(p01, p23)); // 0,1,2,3, 0,1,2,3 ++ } else { ++ // Note to myself: this is much faster than using _mm256_hadd_epi32() ++ auto p01 = _mm256_add_epi32(_mm256_unpacklo_epi32(p0, p1), _mm256_unpackhi_epi32(p0, p1)); // 0,1, 0,1, 0,1, 0,1 ++ auto p23 = _mm256_add_epi32(_mm256_unpacklo_epi32(p2, p3), _mm256_unpackhi_epi32(p2, p3)); // 2,3, 2,3, 2,3, 2,3 ++ return _mm256_add_epi32(_mm256_unpacklo_epi64(p01, p23), _mm256_unpackhi_epi64(p01, p23)); // 0,1,2,3, 0,1,2,3 ++ } ++ } ++ inline __m256i compute(__m256i x, __m256i y) const { return dot.compute(x, y); } ++}; ++ ++template struct Sum4q4 { ++ inline __m256i compute(const __m256i * qx, const Q8 * y) const { ++ const Q8x4 * y4 = (const Q8x4 *)y; ++ auto p0 = _mm256_maddubs_epi16(qx[0], _mm256_loadu_si256((const __m256i *)y4->qs+0)); // 16x block 0 ++ auto p1 = _mm256_maddubs_epi16(qx[1], _mm256_loadu_si256((const __m256i *)y4->qs+1)); // 16x block 1 ++ auto p2 = _mm256_maddubs_epi16(qx[2], _mm256_loadu_si256((const __m256i *)y4->qs+2)); // 16x block 2 ++ auto p3 = _mm256_maddubs_epi16(qx[3], _mm256_loadu_si256((const __m256i *)y4->qs+3)); // 16x block 3 ++ auto p01 = _mm256_add_epi16(_mm256_unpacklo_epi32(p0, p1), _mm256_unpackhi_epi32(p0, p1)); // 0,0, 1,1, 0,0, 1,1, 0,0, 1,1, 0,0, 1,1 ++ auto p23 = _mm256_add_epi16(_mm256_unpacklo_epi32(p2, p3), _mm256_unpackhi_epi32(p2, p3)); // 2,2, 3,3, 2,2, 3,3, 2,2, 3,3, 2,2, 3,3 ++ auto p0123 = _mm256_add_epi16(_mm256_unpacklo_epi64(p01, p23), _mm256_unpackhi_epi64(p01, p23)); // 0,0, 1,1, 2,2, 3,3, 0,0, 1,1, 2,2, 3,3 ++ return _mm256_madd_epi16(_mm256_set1_epi16(1), p0123); ++ } ++ inline __m256i compute(__m256i x, __m256i y) const { return _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(x, y)); } ++}; ++ ++inline __m256 convert_scales(const uint16_t * scales) { ++ auto aux_d = _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)scales)), 16)); ++ auto aux_m = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(scales+4)))); ++ return _mm256_set_m128(_mm_mul_ps(aux_d, aux_m), aux_d); ++} ++ ++inline __m128 convert_scales_s(const uint16_t * scales) { ++ return _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)scales)), 16)); ++} ++ ++struct ScaleHelperQ8_0 { ++ inline __m128 prepare4(const block_q8_0 * y) { ++ const block_q8_0_x4 * y4 = (const block_q8_0_x4 *)y; ++ return _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)y4->d)); ++ } ++ inline __m128 prepare4(__m128 other_scales, const block_q8_0 * y) { ++ return _mm_mul_ps(other_scales, prepare4(y)); ++ } ++ template inline float prepare1(const Q * y) const { return GGML_FP16_TO_FP32(y->d); } ++ template inline float prepare1(float d, const Q * y) const { return d*prepare1(y); } ++}; ++ ++struct ScaleHelperQ_0 { ++ ggml_half scales8[4]; ++ template ++ inline __m128 prepare4(const Q * y) { ++ for (int j = 0; j < 4; ++j) scales8[j] = y[j].d; ++ return _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)scales8)); ++ } ++ template ++ inline __m128 prepare4(__m128 other_scales, const Q * y) { ++ return _mm_mul_ps(other_scales, prepare4(y)); ++ } ++ template inline float prepare1(const Q * y) const { return GGML_FP16_TO_FP32(y->d); } ++ template inline float prepare1(float d, const Q * y) const { return d*prepare1(y); } ++}; ++ ++struct ScaleHelperQ8_2S { ++ template ++ inline __m128 prepare4(const Q * y) { ++ const block_q8_2_x4 * y4 = (const block_q8_2_x4 *)y; ++ return convert_scales_s((const uint16_t *)y4->d); ++ } ++ template ++ inline __m128 prepare4(__m128 other_scales, const Q * y) { ++ return _mm_mul_ps(other_scales, prepare4(y)); ++ } ++ template static inline float prepare1(const Q * y) { return GGML_BF16_TO_FP32(ggml_bf16_t{y->d}); } ++ template static inline float prepare1(float d, const Q * y) { return d*prepare1(y); } ++}; ++ ++struct ScaleHelperQ_0_MXFP4 { ++ float scales[4]; ++ template ++ inline __m128 prepare4(const Q * y) { ++ for (int j = 0; j < 4; ++j) scales[j] = GGML_E8M0_TO_FP32_HALF(y[j].e); ++ return _mm_loadu_ps(scales); ++ } ++ template ++ inline __m128 prepare4(__m128 other_scales, const Q * y) { ++ return _mm_mul_ps(other_scales, prepare4(y)); ++ } ++ template inline float prepare1(const Q * y) const { return GGML_E8M0_TO_FP32_HALF(y->e); } ++ template inline float prepare1(float d, const Q * y) const { return d*prepare1(y); } ++}; ++ ++template ++struct ScaleHelperQ_0_1 { ++ ggml_half scales8[4]; ++ template ++ inline __m256 prepare4(const Q * y) { ++ for (int j = 0; j < 4; ++j) scales8[j] = y[j].d; ++ auto s4 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)scales8)); ++ return _mm256_set_m128(_mm_mul_ps(s4, min), s4); ++ } ++ template ++ inline __m256 prepare4(__m256 other_scales, const Q * y) { ++ return _mm_mul256_ps(other_scales, prepare4(y)); ++ } ++ template inline std::pair prepare1(const Q * y) const { ++ float d = GGML_FP16_TO_FP32(y->d); ++ return std::make_pair(d, -d*float(min_value)); ++ } ++ std::pair inline prepare1(const std::pair& dm, const block_q8_1 * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->s)); ++ } ++ const __m128 min = _mm_set1_ps(float(-min_value)); ++}; ++ ++template ++struct ScaleHelperQ_0_1_MXFP4 { ++ float scales[4]; ++ template ++ inline __m256 prepare4(const Q * y) { ++ for (int j = 0; j < 4; ++j) scales[j] = GGML_E8M0_TO_FP32_HALF(y[j].e); ++ auto s4 = _mm_loadu_ps(scales); ++ return _mm256_set_m128(_mm_mul_ps(s4, min), s4); ++ } ++ template ++ inline __m256 prepare4(__m256 other_scales, const Q * y) { ++ return _mm_mul256_ps(other_scales, prepare4(y)); ++ } ++ template inline std::pair prepare1(const Q * y) const { ++ float d = GGML_E8M0_TO_FP32_HALF(y->e); ++ return std::make_pair(d, -d*float(min_value)); ++ } ++ std::pair inline prepare1(const std::pair& dm, const block_q8_1 * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->s)); ++ } ++ const __m128 min = _mm_set1_ps(float(-min_value)); ++}; ++ ++struct ScaleHelperQ8_1 { ++ template ++ inline __m256 prepare4(const Q * y) { ++ const block_q8_1_x4 * y4 = (const block_q8_1_x4 *)y; ++ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)y4->d)); ++ } ++ template ++ inline __m256 prepare4(__m256 other_scales, const Q * y) { ++ return _mm256_mul_ps(other_scales, prepare4(y)); ++ } ++ template inline std::pair prepare1(const Q * y) const { ++ return std::make_pair(GGML_FP16_TO_FP32(y->d), GGML_FP16_TO_FP32(y->m)); ++ } ++ template inline std::pair prepare1(const std::pair& dm, const Q * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->m)); ++ } ++ std::pair inline prepare1(const std::pair& dm, const block_q8_1 * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->s)); ++ } ++}; ++ ++struct ScaleHelperQ8_2 { ++ template ++ inline __m256 prepare4(const Q * y) { ++ const block_q8_2_x4 * y4 = (const block_q8_2_x4 *)y; ++ return convert_scales((const uint16_t *)y4->d); ++ } ++ template ++ inline __m256 prepare4(__m256 other_scales, const Q * y) { ++ return _mm256_mul_ps(other_scales, prepare4(y)); ++ } ++ template static inline std::pair prepare1(const Q * y) { ++ float d = GGML_BF16_TO_FP32(ggml_bf16_t{y->d}); ++ int16_t m = *(const int16_t *)&y->s; ++ return std::make_pair(d, d*m); ++ } ++ static inline std::pair prepare1(const std::pair& dm, const block_q8_2 * y) { ++ auto d = prepare1(y); ++ return std::make_pair(dm.first*d.first, dm.second*d.second); ++ } ++}; ++ ++struct ScaleHelperQ_1 { ++ uint32_t scales8[4]; ++ const __m128i shuffle = _mm_set_epi16(0x0f0e, 0x0b0a, 0x0706, 0x0302, 0x0d0c, 0x0908, 0x0504, 0x0100); ++ ++ template ++ inline __m256 prepare4(const Q * y) { ++ for (int j = 0; j < 4; ++j) { ++ // it is slightly faster to directly dereference (const uint32 *)&y[j].d, but some compilers ++ // complain that this breaks strict-aliasing rules. ++ memcpy(scales8 + j, &y[j].d, sizeof(uint32_t)); ++ } ++ return _mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)scales8), shuffle)); ++ } ++ ++ template ++ inline __m256 prepare4(__m256 other_scales, const Q * y) { ++ return _mm256_mul_ps(other_scales, prepare4(y)); ++ } ++ ++ template inline std::pair prepare1(const Q * y) const { ++ return std::make_pair(GGML_FP16_TO_FP32(y->d), GGML_FP16_TO_FP32(y->m)); ++ } ++ template inline std::pair prepare1(const std::pair& dm, const Q * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->m)); ++ } ++ std::pair inline prepare1(const std::pair& dm, const block_q8_1 * y) const { ++ return std::make_pair(dm.first*GGML_FP16_TO_FP32(y->d), dm.second*GGML_FP16_TO_FP32(y->s)); ++ } ++}; ++ ++struct MinusType0 { ++ inline __m256 compute(__m128 d, int) const { return _mm256_set_m128(d, d); } ++ inline float compute(float d, int) const { return d; } ++ inline float result(__m256 acc, int) const { return hsum_float_8(acc); } ++ inline __m256 vresult(__m256 acc, int) const { return acc; } ++}; ++ ++template struct MinusType1 { ++ __m128 accm[nrc_y]; ++ MinusType1() { for (int iy = 0; iy < nrc_y; ++iy) accm[iy] = _mm_setzero_ps(); } ++ inline __m256 compute(__m256 dm, int iy) { ++ const __m128 d = _mm256_castps256_ps128(dm); ++ const __m128 m = _mm256_extractf128_ps(dm, 1); ++ accm[iy] = _mm_add_ps(accm[iy], m); ++ return _mm256_set_m128(d, d); ++ } ++ inline float compute(const std::pair& dm, int iy) { ++ accm[iy] = _mm_add_ps(accm[iy], _mm_set1_ps(dm.second*0.25f)); ++ return dm.first; ++ } ++ inline float result(__m256 acc, int iy) const { ++ const __m128 sum = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); ++ return hsum_float_4(_mm_add_ps(sum, accm[iy])); ++ } ++ inline __m256 vresult(__m256 acc, int iy) const { ++ return _mm256_add_ps(acc, _mm256_insertf128_ps(_mm256_setzero_ps(), accm[iy], 0)); ++ } ++}; ++ ++template struct AccumT { ++ __m256 acc[nrc_y]; ++ Minus accm; ++ AccumT() { for (int iy = 0; iy < nrc_y; ++iy) acc[iy] = _mm256_setzero_ps(); } ++ template ++ inline void compute(int nb, Unpacker& unp, Scales& scales, Sum& sum, const Q8 ** y, const DataInfo& info, int ix) { ++ auto qx = unp.quants(); ++ __m256 dall[nrc_y]; ++ for (int i = 0; i < nb/4; ++i) { ++ auto other_scales = unp.set_block_4(i); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto s12 = scales.prepare4(other_scales, y[iy] + 4*i); ++ dall[iy] = accm.compute(s12, iy); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto pall = sum.compute(qx, y[iy] + 4*i); ++ acc[iy] = _mm256_fmadd_ps(dall[iy], _mm256_cvtepi32_ps(pall), acc[iy]); ++ } ++ } ++ if (!is_multiple_of_4) { ++ for (int i = 4*(nb/4); i < nb; ++i) { ++ auto other_scales = unp.set_block(i); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto s12 = scales.prepare1(other_scales, y[iy] + i); ++ auto d = accm.compute(s12, iy); ++ const __m256i p0 = sum.compute(qx[0], _mm256_loadu_si256((const __m256i *)y[iy][i].qs)); ++ acc[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(p0), acc[iy]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, accm.result(acc[iy], iy)); ++ } ++ } ++ template ++ inline void compute(int nb, Unpacker& unp, Scales& scales, Sum& sum, const Q8 ** y, __m256 * result) { ++ auto qx = unp.quants(); ++ __m256 dall[nrc_y]; ++ for (int i = 0; i < nb/4; ++i) { ++ auto other_scales = unp.set_block_4(i); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto s12 = scales.prepare4(other_scales, y[iy] + 4*i); ++ dall[iy] = accm.compute(s12, iy); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto pall = sum.compute(qx, y[iy] + 4*i); ++ acc[iy] = _mm256_fmadd_ps(dall[iy], _mm256_cvtepi32_ps(pall), acc[iy]); ++ } ++ } ++ if (!is_multiple_of_4) { ++ for (int i = 4*(nb/4); i < nb; ++i) { ++ auto other_scales = unp.set_block(i); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto s12 = scales.prepare1(other_scales, y[iy] + i); ++ auto d = accm.compute(s12, iy); ++ const __m256i p0 = sum.compute(qx[0], _mm256_loadu_si256((const __m256i *)y[iy][i].qs)); ++ acc[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(p0), acc[iy]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ result[iy] = accm.vresult(acc[iy], iy); ++ } ++ } ++}; ++ ++template ++using AccumType0 = AccumT; ++ ++template ++using AccumType1 = AccumT, nrc_y, is_multiple_of_4>; ++ ++using Sum4TypeQ80 = Sum4; ++using Sum4TypeQ82 = Sum4; ++using Sum4TypeQ82S = Sum4; ++ ++template ++void mul_mat_qX_q8_Helper(int nb, const void * vx, size_t bx, const DataInfo& info, const Q8 ** y, int nrc_x) { ++ Unpacker unp(vx, bx); ++ typename Unpacker::Sum4T sum4; ++ Scales scales; ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ unp.set_row(ix); ++ AccumType accum; ++ accum.compute(nb, unp, scales, sum4, y, info, ix); ++ } ++} ++ ++template ++void mul_mat_qX_q8_Helper_x2(int nb, const void * vx, size_t bx, const DataInfo& info, const Q8 ** y, int nrc_x) { ++ GGML_ASSERT(nrc_x%2 == 0); ++ Unpacker unp(vx, bx); ++ typename Unpacker::Sum4T sum4; ++ Scales scales; ++ for (int ix = 0; ix < nrc_x; ix += 2) { ++ unp.set_row(ix); ++ AccumType accum; ++ accum.compute(nb, unp, scales, sum4, y, info, ix); ++ } ++} ++ ++template ++void mul_mat_qX_0_q8_0_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%Unpacker::block_size() == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ if constexpr (std::is_same_v) { ++ if (nb%4 == 0) { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_2S, Block, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x); ++ } else { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_2S, Block, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x); ++ } ++ } ++ else { ++ if (nb%4 == 0) { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_0, Block, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x); ++ } else { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_0, Block, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x); ++ } ++ } ++} ++ ++template ++void mul_mat_qX_0_q8_2_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%Unpacker::block_size() == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ if (nb%4 == 0) { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_0, block_q8_0, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } else { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_0, block_q8_0, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } ++} ++ ++template ++void mul_mat_qX_0_q8_0_Tx(int n, const void * vx, size_t bx, const DataInfo& info, int) { ++ static_assert(8%nrc_y == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ Unpacker unp(vx, bx); ++ typename Unpacker::Sum4T sum4; ++ ScaleHelperQ8_2S scales; ++ __m256 result[8]; ++ auto store = [&info, &result] (int ix0) { ++ if constexpr (nrc_y == 1) { ++ info.store(ix0, 0, hsum_float_8x8(result)); ++ } ++ else if constexpr (nrc_y == 2) { ++ auto value = hsum_float_8x8(result); ++ auto value1 = _mm256_extractf128_ps(value, 1); ++ info.store(ix0, 0, _mm_shuffle_ps(_mm256_castps256_ps128(value), value1, 0x88)); ++ info.store(ix0, 1, _mm_shuffle_ps(_mm256_castps256_ps128(value), value1, 0xdd)); ++ } ++ else { ++ float val[8]; ++ _mm256_storeu_ps(val, hsum_float_8x8(result)); ++ for (int iy = 0; iy < nrc_y; ++iy) for (int ix = 0; ix < 8/nrc_y; ++ix) info.store(ix0+ix, iy, val[nrc_y*ix+iy]); ++ } ++ }; ++ if (nb%4 == 0) { ++ for (int ix0 = 0; ix0 < nrc_x; ix0 += 8/nrc_y) { ++ for (int ix = 0; ix < 8/nrc_y; ++ix) { ++ unp.set_row(ix0 + ix); ++ AccumType0 accum; ++ accum.compute(nb, unp, scales, sum4, q8.y, result + nrc_y*ix); ++ } ++ store(ix0); ++ } ++ } else { ++ for (int ix0 = 0; ix0 < nrc_x; ix0 += 8/nrc_y) { ++ for (int ix = 0; ix < 8/nrc_y; ++ix) { ++ unp.set_row(ix0 + ix); ++ AccumType0 accum; ++ accum.compute(nb, unp, scales, sum4, q8.y, result + nrc_y*ix); ++ } ++ store(ix0); ++ } ++ } ++} ++ ++template ++void mul_mat_qX_1_q8_1_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%Unpacker::block_size() == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ if (nb%4 == 0) { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_1, block_q8_1, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } else { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_1, block_q8_1, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } ++} ++ ++template ++void mul_mat_qX_1_q8_2_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ assert(n%Unpacker::block_size() == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ if (nb%4 == 0) { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_2, block_q8_2, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } else { ++ mul_mat_qX_q8_Helper, ScaleHelperQ8_2, block_q8_2, nrc_y>( ++ nb, vx, bx, info, q8.y, nrc_x ++ ); ++ } ++} ++ ++template ++void mul_mat_qX_0_q8_2_Tx(int n, const void * vx, size_t bx, const DataInfo& info, int) { ++ static_assert(8%nrc_y == 0); ++ Q8 q8(info); ++ int nb = n/Unpacker::block_size(); ++ Unpacker unp(vx, bx); ++ typename Unpacker::Sum4T sum4; ++ ScaleHelperQ8_2 scales; ++ __m256 result[8]; ++ auto store = [&info, &result] (int ix0) { ++ if constexpr (nrc_y == 1) { ++ info.store(ix0, 0, hsum_float_8x8(result)); ++ } ++ else if constexpr (nrc_y == 2) { ++ auto value = hsum_float_8x8(result); ++ auto value1 = _mm256_extractf128_ps(value, 1); ++ info.store(ix0, 0, _mm_shuffle_ps(_mm256_castps256_ps128(value), value1, 0x88)); ++ info.store(ix0, 1, _mm_shuffle_ps(_mm256_castps256_ps128(value), value1, 0xdd)); ++ } ++ else { ++ float val[8]; ++ _mm256_storeu_ps(val, hsum_float_8x8(result)); ++ for (int iy = 0; iy < nrc_y; ++iy) for (int ix = 0; ix < 8/nrc_y; ++ix) info.store(ix0+ix, iy, val[nrc_y*ix+iy]); ++ } ++ }; ++ if (nb%4 == 0) { ++ for (int ix0 = 0; ix0 < nrc_x; ix0 += 8/nrc_y) { ++ for (int ix = 0; ix < 8/nrc_y; ++ix) { ++ unp.set_row(ix0 + ix); ++ AccumType1 accum; ++ accum.compute(nb, unp, scales, sum4, q8.y, result + nrc_y*ix); ++ } ++ store(ix0); ++ } ++ } else { ++ for (int ix0 = 0; ix0 < nrc_x; ix0 += 8/nrc_y) { ++ for (int ix = 0; ix < 8/nrc_y; ++ix) { ++ unp.set_row(ix0 + ix); ++ AccumType1 accum; ++ accum.compute(nb, unp, scales, sum4, q8.y, result + nrc_y*ix); ++ } ++ store(ix0); ++ } ++ } ++} ++ ++struct Dequantizer4bit { ++ const __m256i m4 = _mm256_set1_epi8(0xf); ++ inline __m256i dequant(const uint8_t * qs) const { ++ const __m128i aux128 = _mm_loadu_si128((const __m128i *)qs); ++ return _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(aux128, 4), aux128), m4); ++ } ++}; ++ ++struct Q8_0_Dequantizer { ++ inline __m256i dequant(const block_q8_0 * x) const { ++ return _mm256_loadu_si256((const __m256i *)x->qs); ++ } ++}; ++ ++struct Q8_0_1_Dequantizer { ++ inline __m256i dequant(const block_q8_0 * x) const { ++ return _mm256_add_epi8(_mm256_set1_epi8(127), _mm256_loadu_si256((const __m256i *)x->qs)); ++ } ++}; ++ ++struct Q4_0_Dequantizer { ++ Dequantizer4bit b4; ++ const __m256i m8 = _mm256_set1_epi8(-8); ++ inline __m256i dequant(const block_q4_0 * x) const { ++ return _mm256_add_epi8(b4.dequant(x->qs), m8); ++ } ++}; ++ ++struct Q4_0_1_Dequantizer { ++ Dequantizer4bit b4; ++ inline __m256i dequant(const block_q4_0 * x) const { ++ return b4.dequant(x->qs); ++ } ++}; ++ ++struct IQ4_NL_DequantizerU { ++ Dequantizer4bit b4; ++ const __m256i values = load_iq4nl_values_256(); ++ inline __m256i dequant(const block_iq4_nl * x) const { ++ return _mm256_shuffle_epi8(values, b4.dequant(x->qs)); ++ } ++}; ++ ++struct IQ4_NL_DequantizerS { ++ Dequantizer4bit b4; ++ const __m256i values = load_iq4k_values_256(); ++ inline __m256i dequant(const block_iq4_nl * x) const { ++ return _mm256_shuffle_epi8(values, b4.dequant(x->qs)); ++ } ++}; ++ ++//============================= ++static inline __m128i load_unsigned_mxfp4_values_128() { ++ static const uint8_t kvalues_mxfp4_unsigned[16] = {12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0}; ++ return _mm_loadu_si128((const __m128i *)kvalues_mxfp4_unsigned); ++} ++ ++static inline __m256i load_unsigned_mxfp4_values_256() { ++ auto val128 = load_unsigned_mxfp4_values_128(); ++ return MM256_SET_M128I(val128, val128); ++} ++ ++#ifdef HAVE_FANCY_SIMD ++static inline __m512i load_unsigned_mxfp4_values_512() { ++ auto val256 = load_unsigned_mxfp4_values_256(); ++ return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); ++} ++#endif ++ ++static inline __m128i load_mxfp4_values_128() { ++ return _mm_loadu_si128((const __m128i *)kvalues_mxfp4); ++} ++ ++static inline __m256i load_mxfp4_values_256() { ++ auto val128 = load_mxfp4_values_128(); ++ return MM256_SET_M128I(val128, val128); ++} ++ ++struct MXFP4_Dequantizer { ++ Dequantizer4bit b4; ++ const __m256i values = load_unsigned_mxfp4_values_256(); ++ inline __m256i dequant(const block_mxfp4 * x) const { ++ return _mm256_shuffle_epi8(values, b4.dequant(x->qs)); ++ } ++}; ++ ++struct MXFP40_Dequantizer { ++ Dequantizer4bit b4; ++ const __m256i values = load_mxfp4_values_256(); ++ inline __m256i dequant(const block_mxfp4 * x) const { ++ return _mm256_shuffle_epi8(values, b4.dequant(x->qs)); ++ } ++}; ++ ++struct Q4_1_Dequantizer { ++ Dequantizer4bit b4; ++ inline __m256i dequant(const block_q4_1 * x) const { ++ return b4.dequant(x->qs); ++ } ++}; ++ ++struct HBitDequantizer { ++ const __m256i shuffle = _mm256_set_epi64x(0x0303030303030303, 0x0202020202020202, 0x0101010101010101, 0x0000000000000000); ++ const __m256i mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe); ++ const __m256i minus1 = _mm256_set1_epi64x(-1); ++ inline __m256i to_bytes(const uint8_t * bits) const { ++ // Note: Data in all ggml quants is at least 2-byte aligned. ++ // => we can cast to uint16_t and use or on two consecutive entries ++ // which is faster than memcpy ++ const uint16_t * aux16 = (const uint16_t *)bits; ++ const uint32_t aux32 = aux16[0] | (aux16[1] << 16); ++ //uint32_t aux32; memcpy(&aux32, bits, sizeof(uint32_t)); ++ __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(aux32), shuffle); ++ bytes = _mm256_or_si256(bytes, mask); ++ return _mm256_cmpeq_epi8(bytes, minus1); ++ } ++}; ++ ++struct Q5_0_Dequantizer { ++ Dequantizer4bit b4; ++ HBitDequantizer hbit; ++ const __m256i mh = _mm256_set1_epi8((char)0xF0); ++ inline __m256i dequant(const block_q5_0 * x) const { ++ const __m256i vqh = _mm256_andnot_si256(hbit.to_bytes(x->qh), mh); ++ return _mm256_or_si256(b4.dequant(x->qs), vqh); ++ } ++}; ++ ++template ++struct Q5_1_Dequantizer { ++ Dequantizer4bit b4; ++ HBitDequantizer hbit; ++ const __m256i mh = _mm256_set1_epi8(0x10); ++ inline __m256i dequant(const Q5 * x) const { ++ const __m256i vqh = _mm256_and_si256(hbit.to_bytes(x->qh), mh); ++ return _mm256_or_si256(b4.dequant(x->qs), vqh); ++ } ++}; ++struct Q6_0_1_Dequantizer { ++ Dequantizer4bit b4; ++ const __m256i mh = _mm256_set1_epi8(0x30); ++ const __m256i shift1 = _mm256_set_epi64x(0, 2, 0, 4); ++ const __m256i shift2 = _mm256_set_epi64x(2, 0, 0, 0); ++ inline __m256i dequant(const block_q6_0 * x) const { ++ uint64_t aux64; std::memcpy(&aux64, x->qh, 8); ++ auto h256 = _mm256_sllv_epi64(_mm256_set1_epi64x(aux64), shift1); ++ return _mm256_or_si256(b4.dequant(x->qs), _mm256_and_si256(_mm256_srlv_epi64(h256, shift2), mh)); ++ } ++}; ++struct Q6_0_Dequantizer { ++ Q6_0_1_Dequantizer deq; ++ inline __m256i dequant(const block_q6_0 * x) const { ++ return _mm256_add_epi8(deq.dequant(x), _mm256_set1_epi8(-32)); ++ } ++}; ++ ++template ++struct Q_Unpacker { ++ Q_Unpacker(const void * vx, size_t bx) : cx_0((const char *)vx), x((const Q*)cx_0), bx(bx) {} ++ ++ const char * cx_0; ++ const Q * x; ++ size_t bx; ++ ++ Scales scales; ++ Dequantizer deq; ++ ++ __m256i qx[4]; ++ ++ inline const __m256i* quants() const { return qx; } ++ ++ inline void set_row(int ix) { x = (const Q*)(cx_0 + ix*bx); } ++ ++ inline auto set_block_4(int i) { ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = deq.dequant(x + 4*i + j); ++ } ++ return scales.prepare4(x + 4*i); ++ } ++ inline auto set_block(int i) { ++ qx[0] = deq.dequant(x + i); ++ return scales.prepare1(x + i); ++ } ++}; ++ ++struct Q8_0_Unpacker final : public Q_Unpacker { ++ Q8_0_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82S; ++ inline static int block_size() { return QK8_0; } ++}; ++struct Q8_0_1_Unpacker final : public Q_Unpacker, Q8_0_1_Dequantizer> { ++ Q8_0_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK8_0; } ++}; ++struct Q8_0_2_Unpacker final : public Q_Unpacker { ++ Q8_0_2_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK8_0; } ++}; ++struct Q4_0_Unpacker final : public Q_Unpacker { ++ Q4_0_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ80; ++ inline static int block_size() { return QK4_0; } ++}; ++struct Q4_0_1_Unpacker final : public Q_Unpacker, Q4_0_1_Dequantizer> { ++ Q4_0_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ //using Sum4T = Sum4TypeQ82; ++ using Sum4T = Sum4q4; ++ inline static int block_size() { return QK4_0; } ++}; ++struct MXFP4_Unpacker final : public Q_Unpacker, MXFP4_Dequantizer> { ++ MXFP4_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK4_NL; } ++}; ++struct IQ4_NL_UnpackerU final : public Q_Unpacker, IQ4_NL_DequantizerU> { ++ IQ4_NL_UnpackerU(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK4_NL; } ++}; ++struct IQ4_NL_UnpackerS final : public Q_Unpacker { ++ IQ4_NL_UnpackerS(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82S; ++ inline static int block_size() { return QK4_NL; } ++}; ++struct Q5_0_Unpacker final : public Q_Unpacker { ++ Q5_0_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ80; ++ inline static int block_size() { return QK5_0; } ++}; ++struct Q5_0_1_Unpacker final : public Q_Unpacker, Q5_1_Dequantizer> { ++ Q5_0_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK5_0; } ++}; ++struct Q4_1_Unpacker final : public Q_Unpacker { ++ Q4_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK4_1; } ++}; ++struct Q5_1_Unpacker final : public Q_Unpacker> { ++ Q5_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK5_1; } ++}; ++struct Q6_0_1_Unpacker final : public Q_Unpacker, Q6_0_1_Dequantizer> { ++ Q6_0_1_Unpacker(const void * vx, size_t bx) : Q_Unpacker(vx, bx) {} ++ using Sum4T = Sum4TypeQ82; ++ inline static int block_size() { return QK6_0; } ++}; ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_iq4_nl_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm512_set1_epi8(0xf); ++ auto values = load_iq4nl_values_512(); ++ int nb = n / QK4_NL; ++ __m512 acc[2*nrc_y] = {}; ++ __m512i qx[4]; ++ float d8[8*nrc_y]; ++ auto prepare = [&qx, &m4, &values] (const block_iq4_nl_r4& iq4l, const block_iq4_nl_r4& iq4h) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq4l.d)); ++ auto scales1 = _mm256_set_m128(scales128, scales128); ++ scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq4h.d)); ++ auto scales2 = _mm256_set_m128(scales128, scales128); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ auto bits1 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq4l.qs+0)), ++ _mm256_loadu_si256((const __m256i *)iq4h.qs+0), 1); ++ auto bits2 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq4l.qs+1)), ++ _mm256_loadu_si256((const __m256i *)iq4h.qs+1), 1); ++ qx[0] = _mm512_shuffle_epi8(values, _mm512_and_si512(bits1, m4)); ++ qx[1] = _mm512_shuffle_epi8(values, _mm512_and_si512(bits2, m4)); ++ qx[2] = _mm512_shuffle_epi8(values, _mm512_and_si512(_mm512_srli_epi16(bits1, 4), m4)); ++ qx[3] = _mm512_shuffle_epi8(values, _mm512_and_si512(_mm512_srli_epi16(bits2, 4), m4)); ++ return scales; ++ }; ++ auto dot = [&qx] (__m256i y8) { ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y8), y8, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ return sumi; ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_nl_r4 * iq4l = (const block_iq4_nl_r4 *)((const char *)vx + (ix+0)*bx); ++ const block_iq4_nl_r4 * iq4h = (const block_iq4_nl_r4 *)((const char *)vx + (ix+4)*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ _mm256_storeu_ps(d8+8*iy, convert_scales((const uint16_t *)q8.y[iy][ib4].d)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq4l[4*ib4+k], iq4h[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto dy = _mm512_set1_ps(d8[8*iy+k]); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(d8[8*iy+k+4]), acc[2*iy+1]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq4l[ib], iq4h[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_1 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ ggml_bf16_t d, s; d.bits = qy[ib].d; s.bits = qy[ib].s; ++ auto dy = _mm512_set1_ps(GGML_BF16_TO_FP32(d)); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(GGML_BF16_TO_FP32(s)), acc[2*iy+1]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum512 = _mm512_fmadd_ps(_mm512_set1_ps(-64.f), acc[2*iy+1], acc[2*iy+0]); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_ps(); ++ auto sum1 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 0), _mm512_extractf32x4_ps(sum512, 1)); ++ auto sum2 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 2), _mm512_extractf32x4_ps(sum512, 3)); ++ info.store(ix+0, iy, sum1); ++ info.store(ix+4, iy, sum2); ++ } ++ } ++} ++#else ++template ++static void mul_mat_iq4_nl_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++#ifndef HAVE_VNNI256 ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); ++ auto values = MM256_SET_M128I(values128, values128); ++ int nb = n / QK4_NL; ++ __m256 acc[nrc_y] = {}; ++ __m256i qs[4]; ++ float d8[4*nrc_y]; ++ auto prepare = [&qs, &values, &m4] (const block_iq4_nl_r4& iq4) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq4.d)); ++ auto scales = _mm256_set_m128(scales128, scales128); ++ auto bits1 = _mm256_loadu_si256((const __m256i *)iq4.qs+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)iq4.qs+1); ++ qs[0] = _mm256_shuffle_epi8(values, _mm256_and_si256(bits1, m4)); ++ qs[1] = _mm256_shuffle_epi8(values, _mm256_and_si256(bits2, m4)); ++ qs[2] = _mm256_shuffle_epi8(values, _mm256_and_si256(_mm256_srli_epi16(bits1, 4), m4)); ++ qs[3] = _mm256_shuffle_epi8(values, _mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4)); ++ return scales; ++ }; ++#ifdef HAVE_VNNI256 ++ auto dot = [&qs] (__m256i y) { ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, _mm256_sign_epi8(qs[0], qs[0]), _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qs[0])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, _mm256_sign_epi8(qs[1], qs[1]), _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qs[1])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, _mm256_sign_epi8(qs[2], qs[2]), _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qs[2])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, _mm256_sign_epi8(qs[3], qs[3]), _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qs[3])); ++ return sumi; ++ }; ++#else ++ auto dot = [&qs, &m1] (__m256i y) { ++ auto u1 = _mm256_sign_epi8(qs[0], qs[0]); ++ auto u2 = _mm256_sign_epi8(qs[1], qs[1]); ++ auto sumi1 = _mm256_add_epi32( ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(u1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qs[0]))), ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(u2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qs[1])))); ++ u1 = _mm256_sign_epi8(qs[2], qs[2]); ++ u2 = _mm256_sign_epi8(qs[3], qs[3]); ++ auto sumi2 = _mm256_add_epi32( ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(u1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qs[2]))), ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(u2, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qs[3])))); ++ return _mm256_add_epi32(sumi1, sumi2); ++ }; ++#endif ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_iq4_nl_r4 * iq4 = (const block_iq4_nl_r4 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto aux = _mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)q8.y[iy][ib4].d)), 16); ++ _mm_storeu_ps(d8+4*iy, _mm_castsi128_ps(aux)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq4[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[4*iy+k])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq4[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_1 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ ggml_bf16_t d{qy[ib].d}; ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(GGML_BF16_TO_FP32(d))); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ info.store(ix, iy, sum); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++#endif ++ ++inline void prepare_q4_0_quants_avx2(const uint8_t * qs, __m256i * v, const __m256i& m4) { ++ auto bits1 = _mm256_loadu_si256((const __m256i *)qs+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)qs+1); ++ auto bits3 = _mm256_loadu_si256((const __m256i *)qs+2); ++ auto bits4 = _mm256_loadu_si256((const __m256i *)qs+3); ++ v[0] = _mm256_and_si256(bits1, m4); ++ v[1] = _mm256_and_si256(bits2, m4); ++ v[2] = _mm256_and_si256(bits3, m4); ++ v[3] = _mm256_and_si256(bits4, m4); ++ v[4] = _mm256_and_si256(_mm256_srli_epi16(bits1, 4), m4); ++ v[5] = _mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4); ++ v[6] = _mm256_and_si256(_mm256_srli_epi16(bits3, 4), m4); ++ v[7] = _mm256_and_si256(_mm256_srli_epi16(bits4, 4), m4); ++} ++ ++inline __m256i accum_q4_0_quants(const __m256i * v, const int8_t * qs) { ++ auto y4l = _mm_loadu_si128((const __m128i*)qs+0); ++ auto y4h = _mm_loadu_si128((const __m128i*)qs+1); ++ auto yl = MM256_SET_M128I(y4l, y4l); ++ auto yh = MM256_SET_M128I(y4h, y4h); ++#ifdef HAVE_FANCY_SIMD ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, v[0], _mm256_shuffle_epi32(yl, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[1], _mm256_shuffle_epi32(yl, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[2], _mm256_shuffle_epi32(yl, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[3], _mm256_shuffle_epi32(yl, 0xff)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[4], _mm256_shuffle_epi32(yh, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[5], _mm256_shuffle_epi32(yh, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[6], _mm256_shuffle_epi32(yh, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, v[7], _mm256_shuffle_epi32(yh, 0xff)); ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(v[0], _mm256_shuffle_epi32(yl, 0x00)), ++ _mm256_maddubs_epi16(v[1], _mm256_shuffle_epi32(yl, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(v[2], _mm256_shuffle_epi32(yl, 0xaa)), ++ _mm256_maddubs_epi16(v[3], _mm256_shuffle_epi32(yl, 0xff))); ++ auto sumi3 = _mm256_add_epi16(_mm256_maddubs_epi16(v[4], _mm256_shuffle_epi32(yh, 0x00)), ++ _mm256_maddubs_epi16(v[5], _mm256_shuffle_epi32(yh, 0x55))); ++ auto sumi4 = _mm256_add_epi16(_mm256_maddubs_epi16(v[6], _mm256_shuffle_epi32(yh, 0xaa)), ++ _mm256_maddubs_epi16(v[7], _mm256_shuffle_epi32(yh, 0xff))); ++ auto sumi = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_add_epi16(_mm256_add_epi16(sumi1, sumi2), _mm256_add_epi16(sumi3, sumi4))); ++#endif ++ return sumi; ++} ++ ++template ++static void mul_mat_q4_0_r8_q8_2_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ int nb = n / QK4_NL; ++ __m256i v[8]; ++ if constexpr (nrc_y == 1) { ++ union { __m256 vec; float val[8]; } helper; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_nl_r8 * iq4 = (const block_iq4_nl_r8 *)((const char *)vx + ix*bx); ++ auto acc1 = _mm256_setzero_ps(); ++ auto acc2 = _mm256_setzero_ps(); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ helper.vec = convert_scales((const uint16_t *)q8.y[0][ib4].d); ++ for (int k = 0; k < 4; ++k) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[4*ib4+k].d)); ++ prepare_q4_0_quants_avx2(iq4[4*ib4+k].qs, v, m4); ++ auto sumi = accum_q4_0_quants(v, q8.y[0][ib4].qs+32*k); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(helper.val[k])); ++ acc1 = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc1); ++ acc2 = _mm256_fmadd_ps(scales, _mm256_set1_ps(helper.val[k+4]), acc2); ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto qy = (const block_q8_2 *)q8.y[0]; ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ib].d)); ++ prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4); ++ auto sumi = accum_q4_0_quants(v, qy[ib].qs); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8)); ++ acc1 = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc1); ++ acc2 = _mm256_fmadd_ps(scales, _mm256_set1_ps(m8), acc2); ++ } ++ acc1 = _mm256_fmadd_ps(acc2, _mm256_set1_ps(-8.f), acc1); ++ info.store(ix, 0, acc1); ++ } ++ } ++ else { ++ __m256 acc[nrc_y] = {}; ++ float d8[8*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_iq4_nl_r8 * iq4 = (const block_iq4_nl_r8 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ { ++ __m256 d4[4]; ++ for (int k = 0; k < 4; ++k) { ++ d4[k] = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[4*ib4+k].d)); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = convert_scales((const uint16_t *)q8.y[iy][ib4].d); ++ _mm256_storeu_ps(d8 + 8*iy, scales); ++ auto m4 = _mm256_extractf128_ps(scales, 1); ++ auto m8 = _mm256_set_m128(m4, m4); ++ auto sumf = _mm256_mul_ps(d4[0], _mm256_shuffle_ps(m8, m8, 0x00)); ++ sumf = _mm256_fmadd_ps(d4[1], _mm256_shuffle_ps(m8, m8, 0x55), sumf); ++ sumf = _mm256_fmadd_ps(d4[2], _mm256_shuffle_ps(m8, m8, 0xaa), sumf); ++ sumf = _mm256_fmadd_ps(d4[3], _mm256_shuffle_ps(m8, m8, 0xff), sumf); ++ acc[iy] = _mm256_fmadd_ps(sumf, _mm256_set1_ps(-8.f), acc[iy]); ++ } ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[4*ib4+k].d)); ++ prepare_q4_0_quants_avx2(iq4[4*ib4+k].qs, v, m4); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = accum_q4_0_quants(v, q8.y[iy][ib4].qs+32*k); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[8*iy+k])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ib].d)); ++ auto scales_m = _mm256_mul_ps(scales, _mm256_set1_ps(-8.f)); ++ prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = accum_q4_0_quants(v, qy[ib].qs); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8)); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(scales_m, _mm256_set1_ps(m8), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q4_0_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ if constexpr (nrc_y == 1) { ++ mul_mat_q4_0_r8_q8_2_avx2<1>(n, vx, bx, info, nrc_x); ++ return; ++ } ++ GGML_ASSERT(nrc_x%16 == 0); ++ Q8 q8(info); ++ auto m4 = _mm512_set1_epi8(0xf); ++ int nb = n / QK4_NL; ++ __m512 acc[2*nrc_y] = {}; ++ __m512i qx[8]; ++ auto prepare = [&qx, &m4] (const block_iq4_nl_r8& iq4l, const block_iq4_nl_r8& iq4h) { ++ auto scales1 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4l.d)); ++ auto scales2 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4h.d)); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ for (int j = 0; j < 4; ++j) { ++ auto bits = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq4l.qs+j)), ++ _mm256_loadu_si256((const __m256i *)iq4h.qs+j), 1); ++ qx[j+0] = _mm512_and_si512(bits, m4); ++ qx[j+4] = _mm512_and_si512(_mm512_srli_epi16(bits, 4), m4); ++ } ++ return scales; ++ }; ++ auto dot = [&qx] (const int8_t * qy) { ++ auto y4l = _mm_loadu_si128((const __m128i*)qy+0); ++ auto y4h = _mm_loadu_si128((const __m128i*)qy+1); ++ auto y8l = MM256_SET_M128I(y4l, y4l); ++ auto y8h = MM256_SET_M128I(y4h, y4h); ++ auto yl = _mm512_inserti32x8(_mm512_castsi256_si512(y8l), y8l, 1); ++ auto yh = _mm512_inserti32x8(_mm512_castsi256_si512(y8h), y8h, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0xff))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[4], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[5], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[6], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[7], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0xff))); ++ return sumi; ++ }; ++ float d8[8*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 16) { ++ const block_iq4_nl_r8 * iq4l = (const block_iq4_nl_r8 *)((const char *)vx + (ix+0)*bx); ++ const block_iq4_nl_r8 * iq4h = (const block_iq4_nl_r8 *)((const char *)vx + (ix+8)*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ _mm256_storeu_ps(d8+8*iy, convert_scales((const uint16_t *)q8.y[iy][ib4].d)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq4l[4*ib4+k], iq4h[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(q8.y[iy][ib4].qs+32*k); ++ auto dy = _mm512_set1_ps(d8[8*iy+k]); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(d8[8*iy+k+4]), acc[2*iy+1]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq4l[ib], iq4h[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_1 *)q8.y[iy]; ++ auto sumi = dot(qy[ib].qs); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto dy = _mm512_set1_ps(d8); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(m8), acc[2*iy+1]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm512_fmadd_ps(_mm512_set1_ps(-8.f), acc[2*iy+1], acc[2*iy+0]); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_ps(); ++ info.store(ix, iy, sum); ++ } ++ } ++} ++#else ++template ++static void mul_mat_q4_0_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ mul_mat_q4_0_r8_q8_2_avx2(n, vx, bx, info, nrc_x); ++} ++#endif ++ ++template ++static void mul_mat_q5_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ auto m5 = _mm256_set1_epi8(0x10); ++#ifndef HAVE_FANCY_SIMD ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ auto mscale = _mm256_set_m128(_mm_set1_ps(-8.f), _mm_set1_ps(1.f)); ++ int nb = n / QK5_0; ++ __m256 acc[nrc_y] = {}; ++ __m256i qx[4]; ++ float d8[8*nrc_y]; ++ auto prepare = [&qx, &m4, &m5] (const block_q5_0_r4& iq5) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq5.d)); ++ auto scales = _mm256_set_m128(scales128, scales128); ++ auto bits1 = _mm256_loadu_si256((const __m256i *)iq5.qs+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)iq5.qs+1); ++ auto hbits = _mm_loadu_si128((const __m128i *)iq5.qh); ++ auto hb = MM256_SET_M128I(_mm_srli_epi16(hbits, 1), hbits); ++ qx[0] = _mm256_or_si256(_mm256_and_si256(bits1, m4), _mm256_and_si256(_mm256_slli_epi16(hb, 4), m5)); ++ qx[1] = _mm256_or_si256(_mm256_and_si256(bits2, m4), _mm256_and_si256(_mm256_slli_epi16(hb, 2), m5)); ++ qx[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits1, 4), m4), _mm256_and_si256(hb, m5)); ++ qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4), _mm256_and_si256(_mm256_srli_epi16(hb, 2), m5));; ++ return scales; ++ }; ++#ifdef HAVE_FANCY_SIMD ++ auto dot = [&qx] (__m256i y) { ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ return sumi; ++ }; ++#else ++ auto dot = [&qx, &m1] (__m256i y) { ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ auto sumi = _mm256_madd_epi16(m1, _mm256_add_epi16(sumi1, sumi2)); ++ return sumi; ++ }; ++#endif ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q5_0_r4 * iq5 = (const block_q5_0_r4 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = convert_scales((const uint16_t *)q8.y[iy][ib4].d); ++ _mm256_storeu_ps(d8 + 8*iy, _mm256_mul_ps(mscale, scales)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq5[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[8*iy+k])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_set1_ps(d8[8*iy+k+4]), acc[iy]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq5[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8)); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_set1_ps(-8.f*m8), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ info.store(ix, iy, sum); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q5_0_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ if constexpr (nrc_y == 1) { ++ mul_mat_q5_0_r4_q8_2_avx2<1>(n, vx, bx, info, nrc_x); ++ } else { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm512_set1_epi8(0xf); ++ auto m5 = _mm512_set1_epi8(0x10); ++ int nb = n / QK5_0; ++ __m512 acc[2*nrc_y] = {}; ++ __m512i qx[4]; ++ float d8[8*nrc_y]; ++ auto prepare = [&qx, &m4, &m5] (const block_q5_0_r4& iq5l, const block_q5_0_r4& iq5h) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq5l.d)); ++ auto scales1 = _mm256_set_m128(scales128, scales128); ++ scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq5h.d)); ++ auto scales2 = _mm256_set_m128(scales128, scales128); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ auto bits1 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq5l.qs+0)), ++ _mm256_loadu_si256((const __m256i *)iq5h.qs+0), 1); ++ auto bits2 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq5l.qs+1)), ++ _mm256_loadu_si256((const __m256i *)iq5h.qs+1), 1); ++ auto hbits1 = _mm_loadu_si128((const __m128i *)iq5l.qh); ++ auto hbits2 = _mm_loadu_si128((const __m128i *)iq5h.qh); ++ auto hb1 = MM256_SET_M128I(_mm_srli_epi16(hbits1, 1), hbits1); ++ auto hb2 = MM256_SET_M128I(_mm_srli_epi16(hbits2, 1), hbits2); ++ auto hb = _mm512_inserti32x8(_mm512_castsi256_si512(hb1), hb2, 1); ++ qx[0] = _mm512_or_si512(_mm512_and_si512(bits1, m4), _mm512_and_si512(_mm512_slli_epi16(hb, 4), m5)); ++ qx[1] = _mm512_or_si512(_mm512_and_si512(bits2, m4), _mm512_and_si512(_mm512_slli_epi16(hb, 2), m5)); ++ qx[2] = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(bits1, 4), m4), _mm512_and_si512(hb, m5)); ++ qx[3] = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(bits2, 4), m4), _mm512_and_si512(_mm512_srli_epi16(hb, 2), m5)); ++ return scales; ++ }; ++ auto dot = [&qx] (__m256i y8) { ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y8), y8, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ return sumi; ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q5_0_r4 * iq5l = (const block_q5_0_r4 *)((const char *)vx + (ix+0)*bx); ++ const block_q5_0_r4 * iq5h = (const block_q5_0_r4 *)((const char *)vx + (ix+4)*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ _mm256_storeu_ps(d8+8*iy, convert_scales((const uint16_t *)q8.y[iy][ib4].d)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq5l[4*ib4+k], iq5h[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto dy = _mm512_set1_ps(d8[8*iy+k]); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(d8[8*iy+k+4]), acc[2*iy+1]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq5l[ib], iq5h[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto dy = _mm512_set1_ps(d8); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(m8), acc[2*iy+1]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum512 = _mm512_fmadd_ps(_mm512_set1_ps(-8.f), acc[2*iy+1], acc[2*iy+0]); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_ps(); ++ auto sum1 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 0), _mm512_extractf32x4_ps(sum512, 1)); ++ auto sum2 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 2), _mm512_extractf32x4_ps(sum512, 3)); ++ info.store(ix+0, iy, sum1); ++ info.store(ix+4, iy, sum2); ++ } ++ } ++ } ++} ++#else ++template ++static void mul_mat_q5_0_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ mul_mat_q5_0_r4_q8_2_avx2(n, vx, bx, info, nrc_x); ++} ++#endif ++ ++template ++static void mul_mat_q6_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ auto m4 = _mm256_set1_epi8(0xf); ++ auto m6 = _mm256_set1_epi8(0x30); ++ auto mscale = _mm256_set_m128(_mm_set1_ps(-16.f), _mm_set1_ps(1.f)); ++#ifndef HAVE_FANCY_SIMD ++ auto m1 = _mm256_set1_epi16(1); ++#endif ++ int nb = n / QK6_0; ++ __m256 acc[nrc_y] = {}; ++ float d8[8*nrc_y]; ++ __m256i qx[4]; ++ auto prepare = [&qx, &m4, &m6] (const block_q6_0_r4& iq6) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq6.d)); ++ auto scales = _mm256_set_m128(scales128, scales128); ++ auto bits1 = _mm256_loadu_si256((const __m256i *)iq6.qs+0); ++ auto bits2 = _mm256_loadu_si256((const __m256i *)iq6.qs+1); ++ auto hbits = _mm256_loadu_si256((const __m256i *)iq6.qh); ++ qx[0] = _mm256_or_si256(_mm256_and_si256(bits1, m4), _mm256_and_si256(_mm256_slli_epi16(hbits, 4), m6)); ++ qx[1] = _mm256_or_si256(_mm256_and_si256(bits2, m4), _mm256_and_si256(_mm256_slli_epi16(hbits, 2), m6)); ++ qx[2] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits1, 4), m4), _mm256_and_si256(hbits, m6)); ++ qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m6)); ++ return scales; ++ }; ++#ifdef HAVE_FANCY_SIMD ++ auto dot = [&qx] (__m256i y) { ++ auto sumi = _mm256_dpbusd_epi32(_mm256_setzero_si256(), qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ return sumi; ++ }; ++#else ++ auto dot = [&qx, &m1] (__m256i y) { ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ auto sumi = _mm256_add_epi32(_mm256_madd_epi16(m1, sumi1), _mm256_madd_epi16(m1, sumi2)); ++ return sumi; ++ }; ++#endif ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ const block_q6_0_r4 * iq6 = (const block_q6_0_r4 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = convert_scales((const uint16_t *)q8.y[iy][ib4].d); ++ _mm256_storeu_ps(d8 + 8*iy, _mm256_mul_ps(scales, mscale)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq6[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[8*iy+k])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_set1_ps(d8[8*iy+k+4]), acc[iy]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq6[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8)); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(scales, _mm256_set1_ps(-16.f*m8), acc[iy]); ++ } ++ } ++ ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum = _mm_add_ps(_mm256_castps256_ps128(acc[iy]), _mm256_extractf128_ps(acc[iy], 1)); ++ info.store(ix, iy, sum); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q6_0_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ if constexpr (nrc_y == 1) { ++ mul_mat_q6_0_r4_q8_2_avx2<1>(n, vx, bx, info, nrc_x); ++ } else { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m4 = _mm512_set1_epi8(0xf); ++ auto m6 = _mm512_set1_epi8(0x30); ++ int nb = n / QK6_0; ++ __m512 acc[2*nrc_y] = {}; ++ __m512i qx[4]; ++ float d8[8*nrc_y]; ++ auto prepare = [&qx, &m4, &m6] (const block_q6_0_r4& iq6l, const block_q6_0_r4& iq6h) { ++ auto scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq6l.d)); ++ auto scales1 = _mm256_set_m128(scales128, scales128); ++ scales128 = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)iq6h.d)); ++ auto scales2 = _mm256_set_m128(scales128, scales128); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ auto bits1 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq6l.qs+0)), ++ _mm256_loadu_si256((const __m256i *)iq6h.qs+0), 1); ++ auto bits2 = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)iq6l.qs+1)), ++ _mm256_loadu_si256((const __m256i *)iq6h.qs+1), 1); ++ auto hbits1 = _mm256_loadu_si256((const __m256i *)iq6l.qh); ++ auto hbits2 = _mm256_loadu_si256((const __m256i *)iq6h.qh); ++ auto hb = _mm512_inserti32x8(_mm512_castsi256_si512(hbits1), hbits2, 1); ++ qx[0] = _mm512_and_si512(bits1, m4) | _mm512_and_si512(_mm512_slli_epi16(hb, 4), m6); ++ qx[1] = _mm512_and_si512(bits2, m4) | _mm512_and_si512(_mm512_slli_epi16(hb, 2), m6);; ++ qx[2] = _mm512_and_si512(_mm512_srli_epi16(bits1, 4), m4) | _mm512_and_si512(hb, m6); ++ qx[3] = _mm512_and_si512(_mm512_srli_epi16(bits2, 4), m4) | _mm512_and_si512(_mm512_srli_epi16(hb, 2), m6); ++ return scales; ++ }; ++ auto dot = [&qx] (__m256i y8) { ++ auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y8), y8, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0xff))); ++ return sumi; ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q6_0_r4 * iq6l = (const block_q6_0_r4 *)((const char *)vx + (ix+0)*bx); ++ const block_q6_0_r4 * iq6h = (const block_q6_0_r4 *)((const char *)vx + (ix+4)*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = convert_scales((const uint16_t *)q8.y[iy][ib4].d); ++ _mm256_storeu_ps(d8 + 8*iy, scales); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = prepare(iq6l[4*ib4+k], iq6h[4*ib4+k]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)q8.y[iy][ib4].qs+k)); ++ auto dy = _mm512_set1_ps(d8[8*iy+k]); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(d8[8*iy+k+4]), acc[2*iy+1]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = prepare(iq6l[ib], iq6h[ib]); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = dot(_mm256_loadu_si256((const __m256i*)qy[ib].qs)); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto dy = _mm512_set1_ps(d8); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(m8), acc[2*iy+1]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum512 = _mm512_fmadd_ps(_mm512_set1_ps(-16.f), acc[2*iy+1], acc[2*iy+0]); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_ps(); ++ auto sum1 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 0), _mm512_extractf32x4_ps(sum512, 1)); ++ auto sum2 = _mm_add_ps(_mm512_extractf32x4_ps(sum512, 2), _mm512_extractf32x4_ps(sum512, 3)); ++ info.store(ix+0, iy, sum1); ++ info.store(ix+4, iy, sum2); ++ } ++ } ++ } ++} ++#else ++template ++static void mul_mat_q6_0_r4_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ mul_mat_q6_0_r4_q8_2_avx2(n, vx, bx, info, nrc_x); ++} ++#endif ++ ++#ifdef HAVE_FANCY_SIMD ++inline __m512i qx_r8_q8_dot_product(const __m512i * qx, const int8_t * y) { ++ auto y4l = _mm_loadu_si128((const __m128i*)y+0); ++ auto y4h = _mm_loadu_si128((const __m128i*)y+1); ++ auto y8l = MM256_SET_M128I(y4l, y4l); ++ auto y8h = MM256_SET_M128I(y4h, y4h); ++ auto yl = _mm512_inserti32x8(_mm512_castsi256_si512(y8l), y8l, 1); ++ auto yh = _mm512_inserti32x8(_mm512_castsi256_si512(y8h), y8h, 1); ++ auto sumi = _mm512_setzero_si512(); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[0], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[1], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[2], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[3], _mm512_shuffle_epi32(yl, _MM_PERM_ENUM(0xff))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[4], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0x00))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[5], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0x55))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[6], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0xaa))); ++ sumi = _mm512_dpbusd_epi32(sumi, qx[7], _mm512_shuffle_epi32(yh, _MM_PERM_ENUM(0xff))); ++ return sumi; ++} ++inline __m256i qx_r8_q8_dot_product(const __m256i * qx, const int8_t * y) { ++ auto y4l = _mm_loadu_si128((const __m128i*)y+0); ++ auto y4h = _mm_loadu_si128((const __m128i*)y+1); ++ auto yl = MM256_SET_M128I(y4l, y4l); ++ auto yh = MM256_SET_M128I(y4h, y4h); ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(yl, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(yl, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(yl, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(yl, 0xff)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[4], _mm256_shuffle_epi32(yh, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[5], _mm256_shuffle_epi32(yh, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[6], _mm256_shuffle_epi32(yh, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[7], _mm256_shuffle_epi32(yh, 0xff)); ++ return sumi; ++} ++inline __m256i q8_0_r8_dot_product(const uint8_t * x, const int8_t * y, __m256i * qx) { ++ for (int i = 0; i < 8; ++i) { ++ qx[i] = _mm256_add_epi8(_mm256_loadu_si256((const __m256i *)x+i), _mm256_set1_epi8(127)); ++ } ++ return qx_r8_q8_dot_product(qx, y); ++} ++template ++static void mul_mat_q8_0_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%16 == 0); ++ Q8 q8(info); ++ int nb = n / QK8_0; ++ if constexpr (nrc_y == 1) { ++ __m256 acc[2] = {}; ++ __m256i qx[8]; ++ float d8[8]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_0_r8 * iq8 = (const block_q8_0_r8 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ _mm256_storeu_ps(d8, convert_scales((const uint16_t *)q8.y[0][ib4].d)); ++ for (int k = 0; k < 4; ++k) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*ib4+k].d)); ++ auto sumi = q8_0_r8_dot_product((const uint8_t *)iq8[4*ib4+k].qs, q8.y[0][ib4].qs+32*k, qx); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[k])); ++ acc[0] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[0]); ++ acc[1] = _mm256_fmadd_ps(scales, _mm256_set1_ps(d8[k+4]), acc[1]); ++ } ++ } ++ if (4*(nb/4) < nb) { ++ auto qy = (const block_q8_2 *)q8.y[0]; ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[ib].d)); ++ auto sumi = q8_0_r8_dot_product((const uint8_t *)iq8[ib].qs, qy[ib].qs, qx); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8)); ++ acc[0] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[0]); ++ acc[1] = _mm256_fmadd_ps(scales, _mm256_set1_ps(m8), acc[1]); ++ } ++ } ++ info.store(ix, 0, _mm256_fmadd_ps(_mm256_set1_ps(-127.f), acc[1], acc[0])); ++ acc[0] = acc[1] = _mm256_setzero_ps(); ++ } ++ } else { ++ __m512 acc[2*nrc_y] = {}; ++ __m512i qx[8]; ++ float d8[8*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 16) { ++ const block_q8_0_r8 * q8l = (const block_q8_0_r8 *)((const char *)vx + (ix+0)*bx); ++ const block_q8_0_r8 * q8h = (const block_q8_0_r8 *)((const char *)vx + (ix+8)*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ _mm256_storeu_ps(d8+8*iy, convert_scales((const uint16_t *)q8.y[iy][ib4].d)); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales1 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8l[4*ib4+k].d)); ++ auto scales2 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8h[4*ib4+k].d)); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ for (int j = 0; j < 8; ++j) { ++ qx[j] = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)q8l[4*ib4+k].qs+j)), ++ _mm256_loadu_si256((const __m256i *)q8h[4*ib4+k].qs+j), 1); ++ qx[j] = _mm512_add_epi8(qx[j], _mm512_set1_epi8(127)); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = qx_r8_q8_dot_product(qx, q8.y[iy][ib4].qs+32*k); ++ auto dy = _mm512_set1_ps(d8[8*iy+k]); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(d8[8*iy+k+4]), acc[2*iy+1]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales1 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8l[ib].d)); ++ auto scales2 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8h[ib].d)); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales1), scales2, 1); ++ for (int j = 0; j < 8; ++j) { ++ qx[j] = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)q8l[ib].qs+j)), ++ _mm256_loadu_si256((const __m256i *)q8h[ib].qs+j), 1); ++ qx[j] = _mm512_add_epi8(qx[j], _mm512_set1_epi8(127)); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = qx_r8_q8_dot_product(qx, qy[ib].qs); ++ auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib); ++ auto dy = _mm512_set1_ps(d8); ++ acc[2*iy+0] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[2*iy+0]); ++ acc[2*iy+1] = _mm512_fmadd_ps(scales, _mm512_set1_ps(m8), acc[2*iy+1]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sum512 = _mm512_fmadd_ps(_mm512_set1_ps(-127.f), acc[2*iy+1], acc[2*iy+0]); ++ info.store(ix, iy, sum512); ++ acc[2*iy+0] = acc[2*iy+1] = _mm512_setzero_ps(); ++ } ++ } ++ } ++} ++#else ++template ++static void mul_mat_q8_0_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ auto m1 = _mm256_set1_epi16(1); ++ int nb = n / QK8_0; ++ __m256 acc[nrc_y] = {}; ++ float d8[4*nrc_y]; ++ __m256i qx[4], sx[4]; ++ auto dot = [&qx, &sx, &m1] (const int8_t * qy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)qy); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_VNNI256 ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, sx[0], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, sx[1], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, sx[2], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2])); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, sx[3], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3])); ++ return sumi; ++#else ++ auto sumi1 = _mm256_add_epi32( ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(sx[0], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0]))), ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(sx[1], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1]))) ++ ); ++ auto sumi2 = _mm256_add_epi32( ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(sx[2], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xaa), qx[2]))), ++ _mm256_madd_epi16(m1, _mm256_maddubs_epi16(sx[3], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0xff), qx[3]))) ++ ); ++ return _mm256_add_epi32(sumi1, sumi2); ++#endif ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_0_r8 * iq8 = (const block_q8_0_r8 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)q8.y[iy][ib4].d)), 16)); ++ _mm_storeu_ps(d8 + 4*iy, scales); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*ib4+k].d)); ++ __m256i sumi_first[nrc_y]; ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*ib4+k].qs+j); ++ sx[j] = _mm256_sign_epi8(qx[j], qx[j]); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ sumi_first[iy] = dot(q8.y[iy][ib4].qs+32*k); ++ } ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*ib4+k].qs+4+j); ++ sx[j] = _mm256_sign_epi8(qx[j], qx[j]); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = _mm256_add_epi32(sumi_first[iy], dot(q8.y[iy][ib4].qs+32*k+16)); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[4*iy+k])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[ib].d)); ++ __m256i sumi_first[nrc_y]; ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[ib].qs+j); ++ sx[j] = _mm256_sign_epi8(qx[j], qx[j]); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ sumi_first[iy] = dot(qy[ib].qs); ++ } ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[ib].qs+4+j); ++ sx[j] = _mm256_sign_epi8(qx[j], qx[j]); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_2 *)q8.y[iy]; ++ auto sumi = _mm256_add_epi32(sumi_first[iy], dot(qy[ib].qs+16)); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(GGML_BF16_TO_FP32(ggml_bf16_t{qy[ib].d}))); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++#endif ++ ++typedef struct { ++ ggml_half d[16]; ++ uint8_t qs[256]; ++} block_q8_1_r8; ++ ++#ifdef HAVE_FANCY_SIMD ++template ++static void mul_mat_q8_1_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%16 == 0); ++ Q8 q8(info); ++ int nb = n / QK8_0; ++ if constexpr (nrc_y == 1) { ++ __m256 acc[1] = {}; ++ float d8[4]; ++ __m256i qx[4]; ++ auto dot = [&qx] (const int8_t * qy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)qy); ++ auto y = MM256_SET_M128I(y128, y128); ++ auto sumi = _mm256_setzero_si256(); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ return sumi; ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_1_r8 * iq8 = (const block_q8_1_r8 *)((const char *)vx + ix*bx); ++ for (int i4 = 0; i4 < nb/4; ++i4) { ++ { ++ __m256 mx[4]; ++ for (int ib32 = 0; ib32 < 4; ++ib32) mx[ib32] = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*i4+ib32].d+1)); ++ auto scales = _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)q8.y[0][i4].d)), 16)); ++ _mm_storeu_ps(d8, scales); ++ auto bsums4 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[0][i4].d+4)))); ++ bsums4 = _mm_mul_ps(bsums4, scales); ++ auto bsums = _mm256_set_m128(bsums4, bsums4); ++ acc[0] = _mm256_fmadd_ps(mx[0], _mm256_shuffle_ps(bsums, bsums, 0x00), acc[0]); ++ acc[0] = _mm256_fmadd_ps(mx[1], _mm256_shuffle_ps(bsums, bsums, 0x55), acc[0]); ++ acc[0] = _mm256_fmadd_ps(mx[2], _mm256_shuffle_ps(bsums, bsums, 0xaa), acc[0]); ++ acc[0] = _mm256_fmadd_ps(mx[3], _mm256_shuffle_ps(bsums, bsums, 0xff), acc[0]); ++ } ++ for (int ib32 = 0; ib32 < 4; ++ib32) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*i4+ib32].d)); ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*i4+ib32].qs+j); ++ } ++ auto sumi = dot(q8.y[0][i4].qs+32*ib32); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[ib32])); ++ acc[0] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[0]); ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*i4+ib32].qs+4+j); ++ } ++ sumi = dot(q8.y[0][i4].qs+32*ib32+16); ++ d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[ib32])); ++ acc[0] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[0]); ++ } ++ } ++ info.store(ix, 0, acc[0]); ++ acc[0] = _mm256_setzero_ps(); ++ } ++ } else { ++ __m512 acc[nrc_y] = {}; ++ __m512i qx[8]; ++ float d8[4*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 16) { ++ const block_q8_1_r8 * q8l = (const block_q8_1_r8 *)((const char *)vx + (ix+0)*bx); ++ const block_q8_1_r8 * q8h = (const block_q8_1_r8 *)((const char *)vx + (ix+8)*bx); ++ for (int i4 = 0; i4 < nb/4; ++i4) { ++ { ++ __m512 mx[4]; ++ for (int ib32 = 0; ib32 < 4; ++ib32) { ++ auto mx_l = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8l[4*i4+ib32].d+1)); ++ auto mx_h = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8h[4*i4+ib32].d+1)); ++ mx[ib32] = _mm512_insertf32x8(_mm512_castps256_ps512(mx_l), mx_h, 1); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales128 = _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)q8.y[iy][i4].d)), 16)); ++ _mm_storeu_ps(d8 + 4*iy, scales128); ++ auto bsums4 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][i4].d+4)))); ++ bsums4 = _mm_mul_ps(bsums4, scales128); ++ auto bsums256 = _mm256_set_m128(bsums4, bsums4); ++ auto bsums = _mm512_insertf32x8(_mm512_castps256_ps512(bsums256), bsums256, 1); ++ acc[iy] = _mm512_fmadd_ps(mx[0], _mm512_shuffle_ps(bsums, bsums, 0x00), acc[iy]); ++ acc[iy] = _mm512_fmadd_ps(mx[1], _mm512_shuffle_ps(bsums, bsums, 0x55), acc[iy]); ++ acc[iy] = _mm512_fmadd_ps(mx[2], _mm512_shuffle_ps(bsums, bsums, 0xaa), acc[iy]); ++ acc[iy] = _mm512_fmadd_ps(mx[3], _mm512_shuffle_ps(bsums, bsums, 0xff), acc[iy]); ++ } ++ } ++ for (int ib32 = 0; ib32 < 4; ++ib32) { ++ auto scales_l = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8l[4*i4+ib32].d)); ++ auto scales_h = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)q8h[4*i4+ib32].d)); ++ auto scales = _mm512_insertf32x8(_mm512_castps256_ps512(scales_l), scales_h, 1); ++ for (int j = 0; j < 8; ++j) { ++ qx[j] = _mm512_inserti32x8(_mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)q8l[4*i4+ib32].qs+j)), ++ _mm256_loadu_si256((const __m256i *)q8h[4*i4+ib32].qs+j), 1); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = qx_r8_q8_dot_product(qx, q8.y[iy][i4].qs+32*ib32); ++ auto dy = _mm512_set1_ps(d8[4*iy+ib32]); ++ acc[iy] = _mm512_fmadd_ps(_mm512_mul_ps(scales, dy), _mm512_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm512_setzero_ps(); ++ } ++ } ++ } ++} ++#else ++template ++static void mul_mat_q8_1_r8_q8_2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ int nb = n / QK8_0; ++ __m256 acc[nrc_y] = {}; ++ float d8[4*nrc_y]; ++ __m256i qx[4]; ++ auto dot = [&qx] (const int8_t * qy) { ++ auto y128 = _mm_loadu_si128((const __m128i*)qy); ++ auto y = MM256_SET_M128I(y128, y128); ++#ifdef HAVE_VNNI256 ++ auto sumi = _mm256_setzero_si256(); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa)); ++ sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff)); ++ return sumi; ++#else ++ auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[0], _mm256_shuffle_epi32(y, 0x00)), ++ _mm256_maddubs_epi16(qx[1], _mm256_shuffle_epi32(y, 0x55))); ++ auto sumi2 = _mm256_add_epi16(_mm256_maddubs_epi16(qx[2], _mm256_shuffle_epi32(y, 0xaa)), ++ _mm256_maddubs_epi16(qx[3], _mm256_shuffle_epi32(y, 0xff))); ++ return _mm256_add_epi32(_mm256_madd_epi16(_mm256_set1_epi16(1), sumi1), _mm256_madd_epi16(_mm256_set1_epi16(1), sumi2)); ++#endif ++ }; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_1_r8 * iq8 = (const block_q8_1_r8 *)((const char *)vx + ix*bx); ++ for (int i4 = 0; i4 < nb/4; ++i4) { ++ { ++ __m256 mx[4]; ++ for (int ib32 = 0; ib32 < 4; ++ib32) mx[ib32] = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*i4+ib32].d+1)); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto scales = _mm_castsi128_ps(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)q8.y[iy][i4].d)), 16)); ++ _mm_storeu_ps(d8 + 4*iy + 0, scales); ++ auto bsums4 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadl_epi64((const __m128i *)(q8.y[iy][i4].d+4)))); ++ bsums4 = _mm_mul_ps(bsums4, scales); ++ auto bsums = _mm256_set_m128(bsums4, bsums4); ++ acc[iy] = _mm256_fmadd_ps(mx[0], _mm256_shuffle_ps(bsums, bsums, 0x00), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(mx[1], _mm256_shuffle_ps(bsums, bsums, 0x55), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(mx[2], _mm256_shuffle_ps(bsums, bsums, 0xaa), acc[iy]); ++ acc[iy] = _mm256_fmadd_ps(mx[3], _mm256_shuffle_ps(bsums, bsums, 0xff), acc[iy]); ++ } ++ } ++ for (int ib32 = 0; ib32 < 4; ++ib32) { ++ auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq8[4*i4+ib32].d)); ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*i4+ib32].qs+j); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(q8.y[iy][i4].qs+32*ib32); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[4*iy+ib32])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ for (int j = 0; j < 4; ++j) { ++ qx[j] = _mm256_loadu_si256((const __m256i *)iq8[4*i4+ib32].qs+4+j); ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto sumi = dot(q8.y[iy][i4].qs+32*ib32+16); ++ auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8[4*iy+ib32])); ++ acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); ++ } ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, acc[iy]); ++ acc[iy] = _mm256_setzero_ps(); ++ } ++ } ++} ++#endif ++ ++void iqk_convert_q80_q80_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ static_assert(QK4_0 == QK8_0); ++ GGML_ASSERT(n%QK4_0 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ const int nb = n/QK4_0; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ const block_q8_0 * x8[8]; ++ ++ uint32_t block[8]; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ ++ for (int k = 0; k < 8; ++k) x8[k] = (const block_q8_0 *)((const char *)vx + (ix + k)*bx); ++ ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ y[i].d[k] = x8[k][i].d; ++ _mm256_storeu_si256((__m256i *)block, _mm256_loadu_si256((const __m256i *)x8[k][i].qs)); ++ auto qs = (uint32_t *)y[i].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++template ++void iqk_convert_qX_q80_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK4_0 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ const int nb = n/QK8_0; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ const Block * x8[8]; ++ ++ uint32_t block[8]; ++ ++ Dequantizer deq; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ ++ for (int k = 0; k < 8; ++k) x8[k] = (const Block *)((const char *)vx + (ix + k)*bx); ++ ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ if constexpr (std::is_same_v) { ++ y[i].d[k] = GGML_FP32_TO_FP16(GGML_E8M0_TO_FP32_HALF(x8[k][i].e)); ++ } else { ++ y[i].d[k] = x8[k][i].d; ++ } ++ _mm256_storeu_si256((__m256i *)block, deq.dequant(x8[k] + i)); ++ auto qs = (uint32_t *)y[i].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++template ++void iqk_convert_qX_1_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK8_0 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ int nb = n/QK8_0; ++ ++ const Block * x8[8]; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ uint32_t block[8]; ++ ++ Dequantizer deq; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = (const Block *)((const char *)vx + (ix + k)*bx); ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ y[i].d[k+0] = x8[k][i].d; ++ y[i].d[k+8] = x8[k][i].m; ++ _mm256_storeu_si256((__m256i *)block, deq.dequant(x8[k]+i)); ++ auto qs = (uint32_t *)y[i].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++template void set_functions(std::array& funcs) { ++ if constexpr (std::is_same_v || std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0_T, Dequantizer, funcs) ++ } ++ else if constexpr (std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T2(mul_mat_qX_0_q8_0_T, Dequantizer, block_q8_2, funcs) ++ } ++ else if constexpr (std::is_same_v || std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_1_q8_2_T, Dequantizer, funcs) ++ } ++ else if constexpr (std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_1_q8_2_T, Dequantizer, funcs) ++ } ++ else if constexpr (std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T2(mul_mat_qX_0_q8_0_T, Dequantizer, block_q8_2, funcs) ++ } ++ else if constexpr (std::is_same_v || std::is_same_v || ++ std::is_same_v || std::is_same_v || ++ std::is_same_v) { ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_1_q8_2_T, Dequantizer, funcs) ++ } ++} ++ ++} // namespace ++ ++bool iqk_convert_legacy_quants_q8_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ switch (type) { ++ case GGML_TYPE_Q4_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q4_1 : iqk_convert_qX_1_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_1 : iqk_convert_qX_1_q8_1_r8>(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q6_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_IQ4_NL: iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q8_0 : iqk_convert_q80_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_MXFP4 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ default: return false; ++ } ++ return true; ++} ++ ++bool iqk_set_kernels_legacy_quants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16) { ++ ++ if (ne00%QK8_0 != 0) return false; ++ ++ auto expected_typeB = GGML_TYPE_Q8_2_X4; ++ ++ func16 = nullptr; ++ ++ switch (typeA) { ++ case GGML_TYPE_Q4_0: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q4_1: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q5_0: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q5_1: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q6_0: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q8_0: ++#ifdef HAVE_FANCY_SIMD ++ set_functions(kernels); ++#else ++ set_functions(kernels); ++#endif ++ break; ++ case GGML_TYPE_IQ4_NL: ++#ifdef HAVE_FANCY_SIMD ++ set_functions(kernels); ++#else ++ set_functions(kernels); ++#endif ++ break; ++ case GGML_TYPE_MXFP4: ++ set_functions(kernels); ++ break; ++ case GGML_TYPE_Q4_0_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q4_0_r8_q8_2, kernels) ++#ifdef HAVE_FANCY_SIMD ++ func16 = mul_mat_q4_0_r8_q8_2<16>; ++#endif ++ break; ++ case GGML_TYPE_Q5_0_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q5_0_r4_q8_2, kernels) ++ break; ++ case GGML_TYPE_Q6_0_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q6_0_r4_q8_2, kernels) ++ break; ++ case GGML_TYPE_Q8_0_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_0_r8_q8_2, kernels) ++ break; ++ case GGML_TYPE_IQ4_NL_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_iq4_nl_r4_q8_2, kernels) ++ break; ++ case GGML_TYPE_Q8_1: // Note: we are misusing the Q8_1 type for Q8_1_R8 ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_1_r8_q8_2, kernels) ++ break; ++ default: ++ return false; ++ } ++ ++ return ggml_type(typeB) == expected_typeB; ++} ++ ++#else ++// ---------------------------- __aarch64__ ---------------------------------------------- ++ ++namespace { ++ ++template ++inline float16x4_t load_scales_q0(const Block * x, ggml_half * aux) { ++ for (int k = 0; k < 4; ++k) aux[k] = x[k].d; ++ return vld1_f16((const float16_t *)aux); ++} ++ ++template ++inline float16x8_t load_scales_q1(const Block * x, ggml_half * aux) { ++ if constexpr (std::is_same_v) { ++ for (int k = 0; k < 4; ++k) { aux[k] = x[k].d; aux[k+4] = x[k].s; } ++ } else { ++ for (int k = 0; k < 4; ++k) { aux[k] = x[k].d; aux[k+4] = x[k].m; } ++ } ++ return vld1q_f16((const float16_t *)aux); ++} ++ ++struct Q4LegacyBits { ++ template ++ inline void prepare(const Block * x) { ++ for (int i = 0; i < 4; ++i) { ++ auto q4bits = vld1q_u8(x[i].qs); ++ b[2*i+0] = vreinterpretq_s8_u8(vandq_u8(q4bits, m4b)); ++ b[2*i+1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits, 4)); ++ } ++ } ++ inline void prepare1(const uint8_t * qs, int8x16_t * q) const { ++ auto q4bits = vld1q_u8(qs); ++ q[0] = vreinterpretq_s8_u8(vandq_u8(q4bits, m4b)); ++ q[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits, 4)); ++ } ++ inline void prepare1(const uint8_t * qs) { ++ prepare1(qs, b); ++ } ++ const uint8x16_t m4b = vdupq_n_u8(0xf); ++ int8x16_t b[8]; ++}; ++ ++// One would think this commented out version would do better than the one below ++// because it offers more opportunities to execute instructions in parallel. ++// Instead, it runs significantly slower. Why? If the compiler is running out of vector registers ++// cannot it just do the sequential version below on its own? ++//inline int32x4_t sum_4_blocks(const int8x16_t * b, const int8_t * qs) { ++// const auto q8b_1 = vld1q_s8_x2(qs + 0); ++// auto p12 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[0], q8b_1.val[0]), b[1], q8b_1.val[1]); ++// const auto q8b_2 = vld1q_s8_x2(qs + 32); ++// auto p34 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[2], q8b_2.val[0]), b[3], q8b_2.val[1]); ++// auto p1234 = vpaddq_s32(p12, p34); ++// const auto q8b_3 = vld1q_s8_x2(qs + 64); ++// auto p56 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[4], q8b_3.val[0]), b[5], q8b_3.val[1]); ++// const auto q8b_4 = vld1q_s8_x2(qs + 96); ++// auto p78 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[6], q8b_4.val[0]), b[7], q8b_4.val[1]); ++// return vpaddq_s32(p1234, vpaddq_s32(p56, p78)); ++//} ++ ++inline int32x4_t sum_4_blocks(const int8x16_t * b, const int8_t * qs) { ++ auto q8b = vld1q_s8_x2(qs + 0); ++ auto p12 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[0], q8b.val[0]), b[1], q8b.val[1]); ++ q8b = vld1q_s8_x2(qs + 32); ++ auto p34 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[2], q8b.val[0]), b[3], q8b.val[1]); ++ auto p1234 = vpaddq_s32(p12, p34); ++ q8b = vld1q_s8_x2(qs + 64); ++ auto p56 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[4], q8b.val[0]), b[5], q8b.val[1]); ++ q8b = vld1q_s8_x2(qs + 96); ++ auto p78 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b[6], q8b.val[0]), b[7], q8b.val[1]); ++ return vpaddq_s32(p1234, vpaddq_s32(p56, p78)); ++} ++ ++inline int32x4x2_t sum_4_blocks(const int8x16_t * b1, const int8x16_t * b2, const int8_t * qs) { ++ auto q8b = vld1q_s8_x2(qs + 0); ++ auto p12_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b1[0], q8b.val[0]), b1[1], q8b.val[1]); ++ auto p12_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b2[0], q8b.val[0]), b2[1], q8b.val[1]); ++ q8b = vld1q_s8_x2(qs + 32); ++ auto p34_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b1[2], q8b.val[0]), b1[3], q8b.val[1]); ++ auto p34_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b2[2], q8b.val[0]), b2[3], q8b.val[1]); ++ auto p1234_1 = vpaddq_s32(p12_1, p34_1); ++ auto p1234_2 = vpaddq_s32(p12_2, p34_2); ++ q8b = vld1q_s8_x2(qs + 64); ++ auto p56_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b1[4], q8b.val[0]), b1[5], q8b.val[1]); ++ auto p56_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b2[4], q8b.val[0]), b2[5], q8b.val[1]); ++ q8b = vld1q_s8_x2(qs + 96); ++ auto p78_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b1[6], q8b.val[0]), b1[7], q8b.val[1]); ++ auto p78_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), b2[6], q8b.val[0]), b2[7], q8b.val[1]); ++ auto p5678_1 = vpaddq_s32(p56_1, p78_1); ++ auto p5678_2 = vpaddq_s32(p56_2, p78_2); ++ return { vpaddq_s32(p1234_1, p5678_1), vpaddq_s32(p1234_2, p5678_2)}; ++} ++ ++template struct Q80 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q80(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8_0 *)info.src1_row(iy); ++ } ++ ++ inline const int8_t * quant_data(int iy, int i) const { ++ const block_q8_0_x4 * y4 = (const block_q8_0_x4 *)y[iy] + i; ++ return y4->qs; ++ } ++ ++ inline float16x4_t load_scales(int iy, int i) const { ++ const block_q8_0_x4 * y4 = (const block_q8_0_x4 *)y[iy] + i; ++ return vld1_f16((const float16_t *)y4->d); ++ } ++ ++ template ++ inline void process_scales(int i, Dequantizer& deq, float16x4_t * sc16, float32x4_t * /*acc*/) const { ++ auto qx_scales = deq.new_block(i); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8_scales = load_scales(iy, i); ++ sc16[iy] = vmul_f16(qx_scales, q8_scales); ++ } ++ } ++ ++ template ++ inline void process_scales(int i, Dequantizer& deq1, Dequantizer& deq2, float16x4_t * sc16, float32x4_t * /*acc*/) const { ++ auto qx_scales_1 = deq1.new_block(i); ++ auto qx_scales_2 = deq2.new_block(i); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8_scales = load_scales(iy, i); ++ sc16[iy ] = vmul_f16(qx_scales_1, q8_scales); ++ sc16[iy+nrc_y] = vmul_f16(qx_scales_2, q8_scales); ++ } ++ } ++ ++ template ++ inline void process_1_block(int i, Dequantizer& deq, float32x4_t * acc) const { ++ deq.prepare1(i); ++ float d = deq.block_scale(i); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8b = vld1q_s8_x2(y[iy][i].qs); ++ auto p = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b[0], q8b.val[0]), deq.bits.b[1], q8b.val[1]); ++ acc[iy] = vmlaq_f32(acc[iy], vdupq_n_f32(d*GGML_FP16_TO_FP32(y[iy][i].d)), vcvtq_f32_s32(p)); ++ } ++ } ++ ++ const block_q8_0 * y[nrc_y]; ++}; ++ ++template struct Q81 { ++ ++ constexpr static int nrc_y = nrc; ++ ++ Q81(const DataInfo& info) { ++ for (int iy = 0; iy < nrc_y; ++iy) y[iy] = (const block_q8_1 *)info.src1_row(iy); ++ } ++ ++ inline const int8_t * quant_data(int iy, int i) const { ++ const block_q8_1_x4 * y4 = (const block_q8_1_x4 *)y[iy] + i; ++ return y4->qs; ++ } ++ ++ inline float16x8_t load_scales(int iy, int i) const { ++ const block_q8_1_x4 * y4 = (const block_q8_1_x4 *)y[iy] + i; ++ return vld1q_f16((const float16_t *)y4->d); ++ } ++ ++ template ++ inline void process_scales(int i, Dequantizer& deq, float16x4_t * sc16, float32x4_t * acc) const { ++ auto qx_scales = deq.new_block(i); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8_scales = load_scales(iy, i); ++ auto m = vmul_f16(vget_high_f16(qx_scales), vget_high_f16(q8_scales)); ++ acc[iy] = vaddq_f32(acc[iy], vcvt_f32_f16(m)); ++ sc16[iy] = vmul_f16(vget_low_f16(qx_scales), vget_low_f16(q8_scales)); ++ } ++ } ++ ++ template ++ inline void process_scales(int i, Dequantizer& deq1, Dequantizer& deq2, float16x4_t * sc16, float32x4_t * acc) const { ++ auto qx_scales_1 = deq1.new_block(i); ++ auto qx_scales_2 = deq2.new_block(i); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8_scales = load_scales(iy, i); ++ auto q8_scales_l = vget_low_f16(q8_scales); ++ auto q8_scales_h = vget_high_f16(q8_scales); ++ auto m1 = vmul_f16(vget_high_f16(qx_scales_1), q8_scales_h); ++ auto m2 = vmul_f16(vget_high_f16(qx_scales_2), q8_scales_h); ++ acc[iy ] = vaddq_f32(acc[iy ], vcvt_f32_f16(m1)); ++ acc[iy+nrc_y ] = vaddq_f32(acc[iy+nrc_y], vcvt_f32_f16(m2)); ++ sc16[iy ] = vmul_f16(vget_low_f16(qx_scales_1), q8_scales_l); ++ sc16[iy+nrc_y] = vmul_f16(vget_low_f16(qx_scales_2), q8_scales_l); ++ } ++ } ++ ++ template ++ inline void process_1_block(int i, Dequantizer& deq, float32x4_t * acc) const { ++ deq.prepare1(i); ++ float d = GGML_FP16_TO_FP32(deq.x[i].d), m = 0.25f*GGML_FP16_TO_FP32(deq.x[i].m); ++ for (int iy = 0; iy < nrc; ++iy) { ++ auto q8b = vld1q_s8_x2(y[iy][i].qs); ++ auto p = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), deq.bits.b[0], q8b.val[0]), deq.bits.b[1], q8b.val[1]); ++ acc[iy] = vmlaq_f32(acc[iy], vdupq_n_f32(d*GGML_FP16_TO_FP32(y[iy][i].d)), vcvtq_f32_s32(p)); ++ acc[iy] = vaddq_f32(acc[iy], vdupq_n_f32(m*GGML_FP16_TO_FP32(y[iy][i].s))); ++ } ++ } ++ ++ const block_q8_1 * y[nrc_y]; ++}; ++ ++template ++struct BaseLegacyDequantizer { ++ ++ BaseLegacyDequantizer(const void * vx, size_t bx) : vx(vx), x(nullptr), bx(bx) {} ++ ++ inline void new_row(int ix) { x = (const block_q *)((const char *)vx + bx*ix); } ++ ++ Q4LegacyBits bits; ++ ++ const void * vx; ++ const block_q * x; ++ size_t bx; ++}; ++ ++struct DequantizerQ40 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ40(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ q[0] = vaddq_s8(q[0], m8); ++ q[1] = vaddq_s8(q[1], m8); ++ } ++ inline void prepare1(int i) { ++ prepare1(i, bits.b); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ ggml_half aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = x[4*i+k].d; ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vld1_f16((const float16_t *)aux); ++ } ++ ++ inline float block_scale(int i) const { return GGML_FP16_TO_FP32(x[i].d); } ++ ++ const int8x16_t m8 = vdupq_n_s8(-8); ++ //ggml_half aux[4]; ++}; ++ ++struct DequantizerQ60 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ60(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ auto qh8 = vld1_u8(x[i].qh); ++ auto qh = vcombine_u8(vshl_n_u8(qh8, 4), qh8); ++ q[0] = vaddq_s8(vorrq_u8(q[0], vandq_u8(qh, hmask)), m32); ++ q[1] = vaddq_s8(vorrq_u8(q[1], vandq_u8(vshrq_n_u8(qh, 2), hmask)), m32); ++ } ++ inline void prepare1(int i) { ++ prepare1(i, bits.b); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ ggml_half aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = x[4*i+k].d; ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vld1_f16((const float16_t *)aux); ++ } ++ inline float block_scale(int i) const { return GGML_FP16_TO_FP32(x[i].d); } ++ ++ const int8x16_t m32 = vdupq_n_s8(-32); ++ const uint8x16_t hmask = vdupq_n_u8(0x30); ++}; ++ ++struct DequantizerIQ4NL final : public BaseLegacyDequantizer { ++ ++ DequantizerIQ4NL(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ q[0] = vqtbl1q_s8(values, q[0]); ++ q[1] = vqtbl1q_s8(values, q[1]); ++ } ++ inline void prepare1(int i) { ++ prepare1(i, bits.b); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ ggml_half aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = x[4*i+k].d; ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vld1_f16((const float16_t *)aux); ++ } ++ static int8x16_t load_values() { ++ static const int8_t iq4nl_values[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; ++ return vld1q_s8(iq4nl_values); ++ } ++ inline float block_scale(int i) const { return GGML_FP16_TO_FP32(x[i].d); } ++ ++ const int8x16_t values = load_values(); ++}; ++ ++struct DequantizerMXFP4 final : public BaseLegacyDequantizer { ++ ++ DequantizerMXFP4(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ q[0] = vqtbl1q_s8(values, q[0]); ++ q[1] = vqtbl1q_s8(values, q[1]); ++ } ++ inline void prepare1(int i) { ++ prepare1(i, bits.b); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ float aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = GGML_E8M0_TO_FP32_HALF(x[4*i+k].e); ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vcvt_f16_f32(vld1q_f32(aux)); ++ } ++ static int8x16_t load_values() { ++ return vld1q_s8(kvalues_mxfp4); ++ } ++ inline float block_scale(int i) const { return GGML_E8M0_TO_FP32_HALF(x[i].e); } ++ ++ const int8x16_t values = load_values(); ++}; ++ ++struct DequantizerQ41 : public BaseLegacyDequantizer { ++ ++ DequantizerQ41(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i) { ++ bits.prepare1(x[i].qs); ++ } ++ ++ inline float16x8_t new_block(int i) { ++ uint32_t aux32[4]; ++ const uint32_t * s32 = (const uint32_t *)&x[4*i].d; ++ for (int k = 0; k < 4; ++k) { ++ aux32[k] = *s32; s32 += sizeof(block_q4_1)/4; ++ bits.prepare1(x[4*i+k].qs, bits.b + 2*k); ++ } ++ return vreinterpretq_f16_u8(vqtbl1q_u8(vld1q_u8((const uint8_t *)aux32), vreinterpretq_u8_u64(shuffle))); ++ } ++ // Leaving this commented out attempt to be reminded that I already tried this. ++ // It has basically the same performance as the version above. ++ //inline float16x8_t new_block(int i) { ++ // uint32x4_t scales = {}; ++ // const block_q4_1 * xi = x + 4*i; ++ // const uint32_t * s32 = (const uint32_t *)&xi->d; ++ // scales = vsetq_lane_u32(*s32, scales, 0); s32 += sizeof(block_q4_1)/4; ++ // bits.prepare1(xi[0].qs, bits.b + 0); ++ // scales = vsetq_lane_u32(*s32, scales, 1); s32 += sizeof(block_q4_1)/4; ++ // bits.prepare1(xi[1].qs, bits.b + 2); ++ // scales = vsetq_lane_u32(*s32, scales, 2); s32 += sizeof(block_q4_1)/4; ++ // bits.prepare1(xi[2].qs, bits.b + 4); ++ // scales = vsetq_lane_u32(*s32, scales, 3); ++ // bits.prepare1(xi[3].qs, bits.b + 6); ++ // return vreinterpretq_f16_u8(vqtbl1q_u8(vreinterpretq_u8_u32(scales), vreinterpretq_u8_u64(shuffle))); ++ //} ++ ++ const uint64x2_t shuffle = {0x0d0c090805040100, 0x0f0e0b0a07060302}; ++}; ++ ++struct HighBit5Legacy { ++ inline uint8x16_t to_bytes(const uint8_t * qh) const { ++ uint8x16_t h = vqtbl1q_u8(vreinterpretq_u8_u16(vdupq_n_u16(*(const uint16_t *)qh)), shuffle); ++ return vceqq_u8(vandq_u8(h, vreinterpretq_u8_u64(mask)), vreinterpretq_u8_u64(mask)); ++ } ++ inline uint8x16_t to_negated_bytes(const uint8_t * qh) const { ++ uint8x16_t h = vqtbl1q_u8(vreinterpretq_u8_u16(vdupq_n_u16(*(const uint16_t *)qh)), shuffle); ++ return vceqq_u8(vandq_u8(h, vreinterpretq_u8_u64(mask)), vdupq_n_u8(0)); ++ } ++ const uint64x2_t mask = vdupq_n_u64(0x8040201008040201); ++ const uint8x16_t shuffle = vcombine_u8(vdup_n_u8(0), vdup_n_u8(1)); ++}; ++ ++struct DequantizerQ50 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ50(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ auto qh = x[i].qh; ++ q[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[0]), vandq_u8(mh, hbits.to_negated_bytes(qh+0)))); ++ q[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[1]), vandq_u8(mh, hbits.to_negated_bytes(qh+2)))); ++ } ++ inline void prepare1(int i) { ++ prepare1(i, bits.b); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ ggml_half aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = x[4*i+k].d; ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vld1_f16((const float16_t *)aux); ++ } ++ inline float block_scale(int i) const { return GGML_FP16_TO_FP32(x[i].d); } ++ ++ HighBit5Legacy hbits; ++ ++ const uint8x16_t mh = vdupq_n_u8(0xf0); ++ ++}; ++ ++struct DequantizerQ80 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ80(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i) { ++ bits.b[0] = vld1q_s8(x[i].qs); ++ bits.b[1] = vld1q_s8(x[i].qs+16); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ ggml_half aux[4]; ++ for (int k = 0; k < 4; ++k) { ++ aux[k] = x[4*i+k].d; ++ bits.b[2*k+0] = vld1q_s8(x[4*i+k].qs); ++ bits.b[2*k+1] = vld1q_s8(x[4*i+k].qs+16); ++ } ++ return vld1_f16((const float16_t *)aux); ++ } ++ inline float block_scale(int i) const { return GGML_FP16_TO_FP32(x[i].d); } ++ ++}; ++ ++// TODO: handle case where row size is not a multiple of 128 ++struct DequantizerQ80_x4 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ80_x4(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i) { ++ bits.b[0] = vld1q_s8(x[i].qs); ++ bits.b[1] = vld1q_s8(x[i].qs+16); ++ } ++ ++ inline float16x4_t new_block(int i) { ++ auto scale = vld1_f16((const float16_t *)x[i].d); ++ for (int k = 0; k < 4; ++k) { ++ bits.b[2*k+0] = vld1q_s8(x[i].qs+32*k); ++ bits.b[2*k+1] = vld1q_s8(x[i].qs+32*k+16); ++ } ++ return scale; ++ } ++ ++}; ++ ++struct DequantizerQ51 final : public BaseLegacyDequantizer { ++ ++ DequantizerQ51(const void * vx, size_t bx) : BaseLegacyDequantizer(vx, bx) {} ++ ++ inline void prepare1(int i, int8x16_t * q) const { ++ bits.prepare1(x[i].qs, q); ++ auto qh = x[i].qh; ++ q[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[0]), vandq_u8(mh, hbits.to_bytes(qh+0)))); ++ q[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(q[1]), vandq_u8(mh, hbits.to_bytes(qh+2)))); ++ } ++ inline void prepare1(int i) { ++ bits.prepare1(x[i].qs, bits.b); ++ } ++ ++ inline float16x8_t new_block(int i) { ++ uint32_t aux32[4]; ++ const uint32_t * s32 = (const uint32_t *)&x[4*i].d; ++ for (int k = 0; k < 4; ++k) { ++ aux32[k] = *s32; s32 += sizeof(block_q5_1)/4; ++ prepare1(4*i+k, bits.b + 2*k); ++ } ++ return vreinterpretq_f16_u8(vqtbl1q_u8(vld1q_u8((const uint8_t *)aux32), vreinterpretq_u8_u64(shuffle))); ++ } ++ ++ HighBit5Legacy hbits; ++ ++ const uint8x16_t mh = vdupq_n_u8(0x10); ++ const uint64x2_t shuffle = {0x0d0c090805040100, 0x0f0e0b0a07060302}; ++ ++}; ++ ++template ++inline void sum_4(int i, Dequantizer& deq, const Q8& q8, const float16x4_t * sc16, float32x4_t * acc) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto pall = sum_4_blocks(deq.bits.b, q8.quant_data(iy, i)); ++ auto scale = vcvt_f32_f16(sc16[iy]); ++ acc[iy] = vmlaq_f32(acc[iy], scale, vcvtq_f32_s32(pall)); ++ } ++} ++ ++template ++inline void sum_4(int i, Dequantizer& deq1, Dequantizer& deq2, const Q8& q8, const float16x4_t * sc16, float32x4_t * acc) { ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ auto pall = sum_4_blocks(deq1.bits.b, deq2.bits.b, q8.quant_data(iy, i)); ++ auto scale1 = vcvt_f32_f16(sc16[iy]); ++ auto scale2 = vcvt_f32_f16(sc16[iy+Q8::nrc_y]); ++ acc[iy] = vmlaq_f32(acc[iy], scale1, vcvtq_f32_s32(pall.val[0])); ++ acc[iy+Q8::nrc_y] = vmlaq_f32(acc[iy+Q8::nrc_y], scale2, vcvtq_f32_s32(pall.val[1])); ++ } ++} ++ ++template ++inline void mul_mat_qX_Y_q8_Y(int n, Dequantizer& deq, Q8& q8, const DataInfo& info, int nrc_x) { ++ const int nb = n / QK4_1; ++ ++ float16x4_t sc16[Q8::nrc_y]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq.new_row(ix); ++ ++ float32x4_t acc[Q8::nrc_y]; ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb/4; ++i) { ++ q8.process_scales(i, deq, sc16, acc); ++ sum_4(i, deq, q8, sc16, acc); ++ } ++ for (int i = 4*(nb/4); i < nb; ++i) { ++ q8.process_1_block(i, deq, acc); ++ } ++ ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ info.store(ix, iy, vaddvq_f32(acc[iy])); ++ } ++ } ++} ++ ++template ++inline void mul_mat_qX_Y_q8_Y_IK(int n, Dequantizer& deq1, Dequantizer& deq2, Q8& q8, const DataInfo& info, int nrc_x) { ++ const int nb = n / QK4_1; ++ ++ float16x4_t sc16[2*Q8::nrc_y]; ++ float32x4_t acc[2*Q8::nrc_y]; ++ ++ for (int ix = 0; ix < nrc_x; ix += 2) { ++ ++ deq1.new_row(ix+0); ++ deq2.new_row(ix+1); ++ ++ for (int iy = 0; iy < 2*Q8::nrc_y; ++iy) acc[iy] = vdupq_n_f32(0.f); ++ ++ for (int i = 0; i < nb/4; ++i) { ++ q8.process_scales(i, deq1, deq2, sc16, acc); ++ sum_4(i, deq1, deq2, q8, sc16, acc); ++ } ++ //for (int i = 4*(nb/4); i < nb; ++i) { ++ // q8.process_1_block(i, deq, acc); ++ //} ++ ++ for (int iy = 0; iy < Q8::nrc_y; ++iy) { ++ info.store(ix+0, iy, vaddvq_f32(acc[iy])); ++ info.store(ix+1, iy, vaddvq_f32(acc[iy+Q8::nrc_y])); ++ } ++ } ++} ++ ++template ++inline void mul_mat_qX_Y_q8_Y_1(int n, Dequantizer& deq1, Dequantizer& deq2, Q8& q8, const DataInfo& info, int nrc_x) { ++ const int nb = n / QK4_1; ++ ++ float16x4_t sc16[2]; ++ ++ for (int ix = 0; ix < nrc_x; ++ix) { ++ ++ deq1.new_row(ix); ++ deq2.new_row(ix); ++ ++ float32x4_t acc[2] = { vdupq_n_f32(0.f), vdupq_n_f32(0.f) }; ++ ++ for (int i = 0; i < nb/8; ++i) { ++ q8.process_scales(2*i+0, deq1, sc16+0, acc+0); ++ q8.process_scales(2*i+1, deq2, sc16+1, acc+1); ++ sum_4(2*i+0, deq1, q8, sc16+0, acc+0); ++ sum_4(2*i+1, deq2, q8, sc16+1, acc+1); ++ } ++ for (int i = 2*(nb/8); i < nb/4; ++i) { ++ q8.process_scales(i, deq1, sc16, acc); ++ sum_4(i, deq1, q8, sc16, acc); ++ } ++ //for (int i = 4*(nb/4); i < nb; ++i) { ++ // q8.process_1_block(i, deq1, acc); ++ //} ++ ++ info.store(ix, 0, vaddvq_f32(vaddq_f32(acc[0], acc[1]))); ++ } ++} ++ ++template ++static void mul_mat_qX_1_q8_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ Q81 q8(info); ++ if constexpr (nrc_y == 1) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x); ++ } else { ++ if (nrc_x%2 == 0 && n%128 == 0) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ mul_mat_qX_Y_q8_Y_IK(n, deq1, deq2, q8, info, nrc_x); ++ } else { ++ Dequantizer deq(vx, bx); ++ mul_mat_qX_Y_q8_Y(n, deq, q8, info, nrc_x); ++ } ++ } ++} ++ ++template ++static void mul_mat_qX_0_q8_0(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ Q80 q8(info); ++ if constexpr (nrc_y == 1) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x); ++ } else { ++ if (nrc_x%2 == 0 && n%128 == 0) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ mul_mat_qX_Y_q8_Y_IK(n, deq1, deq2, q8, info, nrc_x); ++ } else { ++ Dequantizer deq(vx, bx); ++ mul_mat_qX_Y_q8_Y(n, deq, q8, info, nrc_x); ++ } ++ } ++} ++ ++template ++static void mul_mat_qX_1_q8_1_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ Q81<1> q8(info); ++ mul_mat_qX_Y_q8_Y_1(n, deq1, deq2, q8, info, nrc_x); ++} ++ ++template ++static void mul_mat_qX_0_q8_0_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ Dequantizer deq1(vx, bx), deq2(vx, bx); ++ Q80<1> q8(info); ++ mul_mat_qX_Y_q8_Y(n, deq1, deq2, q8, info, nrc_x); ++} ++ ++template ++void mul_mat_qx_r4_q8_0(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%4 == 0); ++ Q8 q8(info); ++ Dequantizer deq(vx, bx); ++ int nb = n / QK4_NL; ++ int8x16_t qx[8]; ++ float d8[4*nrc_y]; ++ float32x4_t acc[nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 4) { ++ deq.new_row(ix); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ vst1q_f32(d8+4*iy, vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][ib4].d))); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = deq.prepare(4*ib4+k, qx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ib4].qs+32*k); ++ auto sumi = interleaved_dotq(qx, y); ++ auto d4d8 = vmulq_f32(scales, vdupq_n_f32(d8[4*iy+k])); ++ acc[iy] = vfmaq_f32(acc[iy], d4d8, vcvtq_f32_s32(sumi)); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = deq.prepare(ib, qx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_0 *)q8.y[iy]; ++ auto y = vld1q_s8_x2(qy[ib].qs); ++ auto sumi = interleaved_dotq(qx, y); ++ auto d4d8 = vmulq_f32(scales, vdupq_n_f32(GGML_FP16_TO_FP32(qy[ib].d))); ++ acc[iy] = vfmaq_f32(acc[iy], d4d8, vcvtq_f32_s32(sumi)); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix, iy, deq.result(acc[iy])); ++ acc[iy] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++template ++void mul_mat_qx_r8_q8_0(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ Dequantizer deq(vx, bx); ++ int nb = n / QK4_NL; ++ int8x16_t qx[16]; ++ float d8[4*nrc_y]; ++ float32x4_t acc[2*nrc_y] = {}; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ deq.new_row(ix); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ vst1q_f32(d8+4*iy, vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][ib4].d))); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales = deq.prepare(ib4, k, qx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto y = vld1q_s8_x2(q8.y[iy][ib4].qs+32*k); ++ auto sumi1 = interleaved_dotq(qx+0, y); ++ auto sumi2 = interleaved_dotq(qx+8, y); ++ auto dy = vdupq_n_f32(d8[4*iy+k]); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales.val[0], dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales.val[1], dy), vcvtq_f32_s32(sumi2)); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales = deq.prepare(ib, 0, qx); ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_0 *)q8.y[iy]; ++ auto y = vld1q_s8_x2(qy[ib].qs); ++ auto sumi1 = interleaved_dotq(qx+0, y); ++ auto sumi2 = interleaved_dotq(qx+8, y); ++ auto dy = vdupq_n_f32(GGML_FP16_TO_FP32(qy[ib].d)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales.val[0], dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales.val[1], dy), vcvtq_f32_s32(sumi2)); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix+0, iy, deq.result(acc[2*iy+0])); ++ info.store(ix+4, iy, deq.result(acc[2*iy+1])); ++ acc[2*iy] = acc[2*iy+1] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++struct IQ4_NL_R4_Dequantizer { ++ IQ4_NL_R4_Dequantizer(const void * vx, size_t bx) : cx((const char *)vx), bx(bx), values(vld1q_s8(iq4k_values)) {} ++ inline void new_row(int ix) { iq4 = (const block_iq4_nl_r4 *)(cx + ix*bx); } ++ inline float32x4_t prepare(int ib, int8x16_t * qx) const { ++ auto scales = vcvt_f32_f16(vld1_f16((const float16_t *)iq4[ib].d)); ++ auto bits = vld1q_u8_x4(iq4[ib].qs); ++ prepare_iq4_nl_quants(values, m4, bits, qx); ++ return scales; ++ } ++ inline float32x4_t result(float32x4_t acc) const { ++ return acc; ++ } ++ ++ const char * cx; ++ const size_t bx; ++ const block_iq4_nl_r4 * iq4; ++ const uint8x16_t m4 = vdupq_n_u8(0x0f); ++ const int8x16_t values; ++}; ++ ++struct Q4_0_R8_Dequantizer { ++ Q4_0_R8_Dequantizer(const void * vx, size_t bx) : cx((const char *)vx), bx(bx) {} ++ inline void new_row(int ix) { iq4 = (const block_iq4_nl_r8 *)(cx + ix*bx); } ++ inline float32x4x2_t prepare(int ib4, int k, int8x16_t * qx) const { ++ auto scales16 = vld1q_f16((const float16_t *)iq4[4*ib4+k].d); ++ float32x4x2_t scales = { vcvt_f32_f16(vget_low_f16(scales16)), vcvt_f32_f16(vget_high_f16(scales16)) }; ++ for (int j = 0; j < 4; ++j) { ++ auto bits = vld1q_u8_x2(iq4[4*ib4+k].qs + 32*j); ++ bits.val[0] = veorq_u8(m88, bits.val[0]); ++ bits.val[1] = veorq_u8(m88, bits.val[1]); ++ qx[2*j+0] = vshlq_n_u8(bits.val[0], 4); ++ qx[2*j+1] = vandq_u8(bits.val[0], m4); ++ qx[2*j+8] = vshlq_n_u8(bits.val[1], 4); ++ qx[2*j+9] = vandq_u8(bits.val[1], m4); ++ } ++ return scales; ++ } ++ inline float32x4_t result(float32x4_t acc) const { ++ return vmulq_f32(norm, acc); ++ } ++ ++ const char * cx; ++ const size_t bx; ++ const block_iq4_nl_r8 * iq4; ++ const uint8x16_t m4 = vdupq_n_u8(0xf0); ++ const uint8x16_t m88 = vdupq_n_u8(0x88); ++ const float32x4_t norm = vdupq_n_f32(1.f/16); ++}; ++ ++struct Q5_0_R4_Dequantizer { ++ Q5_0_R4_Dequantizer(const void * vx, size_t bx) : cx((const char *)vx), bx(bx) {} ++ inline void new_row(int ix) { iq5 = (const block_q5_0_r4 *)(cx + ix*bx); } ++ inline float32x4_t prepare(int ib, int8x16_t * qx) const { ++ auto scales = vcvt_f32_f16(vld1_f16((const float16_t *)iq5[ib].d)); ++ auto lbits = vld1q_u8_x4(iq5[ib].qs); ++ auto hbits = vld1q_u8(iq5[ib].qh); ++ qx[0] = vaddq_s8(vandq_u8(lbits.val[0], m4) | vandq_u8(vshlq_n_u8(hbits, 4), m5), m16); // 0...3 ++ qx[1] = vaddq_s8(vandq_u8(lbits.val[1], m4) | vandq_u8(vshlq_n_u8(hbits, 3), m5), m16); // 16..19 ++ qx[2] = vaddq_s8(vandq_u8(lbits.val[2], m4) | vandq_u8(vshlq_n_u8(hbits, 2), m5), m16); // 4...7 ++ qx[3] = vaddq_s8(vandq_u8(lbits.val[3], m4) | vandq_u8(vshlq_n_u8(hbits, 1), m5), m16); // 20..23 ++ qx[4] = vaddq_s8(vshrq_n_u8(lbits.val[0], 4)| vandq_u8(hbits, m5), m16); // 8..11 ++ qx[5] = vaddq_s8(vshrq_n_u8(lbits.val[1], 4)| vandq_u8(vshrq_n_u8(hbits, 1), m5), m16); // 24..27 ++ qx[6] = vaddq_s8(vshrq_n_u8(lbits.val[2], 4)| vandq_u8(vshrq_n_u8(hbits, 2), m5), m16); // 12..15 ++ qx[7] = vaddq_s8(vshrq_n_u8(lbits.val[3], 4)| vandq_u8(vshrq_n_u8(hbits, 3), m5), m16); // 28..31 ++ return scales; ++ } ++ inline float32x4_t result(float32x4_t acc) const { ++ return acc; ++ } ++ ++ const char * cx; ++ const size_t bx; ++ const block_q5_0_r4 * iq5; ++ const uint8x16_t m4 = vdupq_n_u8(0x0f); ++ const uint8x16_t m5 = vdupq_n_u8(0x10); ++ const int8x16_t m16 = vdupq_n_s8(-16); ++}; ++ ++struct Q6_0_R4_Dequantizer { ++ Q6_0_R4_Dequantizer(const void * vx, size_t bx) : cx((const char *)vx), bx(bx) {} ++ inline void new_row(int ix) { iq6 = (const block_q6_0_r4 *)(cx + ix*bx); } ++ inline float32x4_t prepare(int ib, int8x16_t * qx) const { ++ auto scales = vcvt_f32_f16(vld1_f16((const float16_t *)iq6[ib].d)); ++ auto lbits = vld1q_u8_x4(iq6[ib].qs); ++ auto hbits = vld1q_u8_x2(iq6[ib].qh); ++ qx[0] = vaddq_s8(vandq_u8(lbits.val[0], m4) | vandq_u8(vshlq_n_u8(hbits.val[0], 4), m6), m32); // 0...3 ++ qx[1] = vaddq_s8(vandq_u8(lbits.val[1], m4) | vandq_u8(vshlq_n_u8(hbits.val[1], 4), m6), m32); // 16..19 ++ qx[2] = vaddq_s8(vandq_u8(lbits.val[2], m4) | vandq_u8(vshlq_n_u8(hbits.val[0], 2), m6), m32); // 4...7 ++ qx[3] = vaddq_s8(vandq_u8(lbits.val[3], m4) | vandq_u8(vshlq_n_u8(hbits.val[1], 2), m6), m32); // 20..23 ++ qx[4] = vaddq_s8(vshrq_n_u8(lbits.val[0], 4)| vandq_u8(hbits.val[0], m6), m32); // 8..11 ++ qx[5] = vaddq_s8(vshrq_n_u8(lbits.val[1], 4)| vandq_u8(hbits.val[1], m6), m32); // 24..27 ++ qx[6] = vaddq_s8(vshrq_n_u8(lbits.val[2], 4)| vandq_u8(vshrq_n_u8(hbits.val[0], 2), m6), m32); // 12..15 ++ qx[7] = vaddq_s8(vshrq_n_u8(lbits.val[3], 4)| vandq_u8(vshrq_n_u8(hbits.val[1], 2), m6), m32); // 28..31 ++ return scales; ++ } ++ inline float32x4_t result(float32x4_t acc) const { ++ return acc; ++ } ++ ++ const char * cx; ++ const size_t bx; ++ const block_q6_0_r4 * iq6; ++ const uint8x16_t m4 = vdupq_n_u8(0x0f); ++ const uint8x16_t m6 = vdupq_n_u8(0x30); ++ const int8x16_t m32 = vdupq_n_s8(-32); ++}; ++ ++inline void qx_0_q8_0_dot(const int8x16_t * qx, const int8_t * qy, int32x4_t& sumi1, int32x4_t& sumi2) { ++ auto y = vld1q_s8_x2(qy); ++ sumi1 = sumi2 = vdupq_n_s32(0); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[0], y.val[0], 0); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[1], y.val[0], 0); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[2], y.val[0], 1); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[3], y.val[0], 1); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[4], y.val[0], 2); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[5], y.val[0], 2); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[6], y.val[0], 3); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[7], y.val[0], 3); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[8+0], y.val[1], 0); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[8+1], y.val[1], 0); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[8+2], y.val[1], 1); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[8+3], y.val[1], 1); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[8+4], y.val[1], 2); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[8+5], y.val[1], 2); ++ sumi1 = vdotq_laneq_s32(sumi1, qx[8+6], y.val[1], 3); ++ sumi2 = vdotq_laneq_s32(sumi2, qx[8+7], y.val[1], 3); ++} ++ ++template ++void mul_mat_q8_0_r8_q8_0(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ int nb = n / QK8_0; ++ float32x4_t acc[2*nrc_y] = {}; ++ int8x16_t qx[16]; ++ float d8[4*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_0_r8 * iq8 = (const block_q8_0_r8 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ vst1q_f32(d8+4*iy, vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][ib4].d))); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales16 = vld1q_f16((const float16_t *)iq8[4*ib4+k].d); ++ auto scales1 = vcvt_f32_f16(vget_low_f16 (scales16)); ++ auto scales2 = vcvt_f32_f16(vget_high_f16(scales16)); ++ for (int j = 0; j < 16; ++j) qx[j] = vld1q_s8(iq8[4*ib4+k].qs + 16*j); ++ int32x4_t sumi1, sumi2; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ qx_0_q8_0_dot(qx, q8.y[iy][ib4].qs+32*k, sumi1, sumi2); ++ auto dy = vdupq_n_f32(d8[4*iy+k]); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales1, dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales2, dy), vcvtq_f32_s32(sumi2)); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales16 = vld1q_f16((const float16_t *)iq8[ib].d); ++ auto scales1 = vcvt_f32_f16(vget_low_f16 (scales16)); ++ auto scales2 = vcvt_f32_f16(vget_high_f16(scales16)); ++ for (int j = 0; j < 16; ++j) qx[j] = vld1q_s8(iq8[ib].qs + 16*j); ++ int32x4_t sumi1, sumi2; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_0 *)q8.y[iy]; ++ qx_0_q8_0_dot(qx, qy[ib].qs, sumi1, sumi2); ++ auto dy = vdupq_n_f32(GGML_FP16_TO_FP32(qy[ib].d)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales1, dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales2, dy), vcvtq_f32_s32(sumi2)); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix+0, iy, acc[2*iy+0]); ++ info.store(ix+4, iy, acc[2*iy+1]); ++ acc[2*iy] = acc[2*iy+1] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++typedef struct { ++ ggml_half d[16]; ++ int8_t qs[256]; ++} block_q8_1_r8; ++ ++template ++void mul_mat_q8_1_r8_q8_1(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { ++ GGML_ASSERT(nrc_x%8 == 0); ++ Q8 q8(info); ++ int nb = n / QK8_0; ++ float32x4_t acc[2*nrc_y] = {}; ++ int8x16_t qx[16]; ++ float d8[8*nrc_y]; ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ const block_q8_1_r8 * iq8 = (const block_q8_1_r8 *)((const char *)vx + ix*bx); ++ for (int ib4 = 0; ib4 < nb/4; ++ib4) { ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ vst1q_f32(d8+8*iy+0, vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][ib4].d+0))); ++ vst1q_f32(d8+8*iy+4, vcvt_f32_f16(vld1_f16((const float16_t *)q8.y[iy][ib4].d+4))); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto scales16 = vld1q_f16((const float16_t *)iq8[4*ib4+k].d); ++ auto scales1 = vcvt_f32_f16(vget_low_f16 (scales16)); ++ auto scales2 = vcvt_f32_f16(vget_high_f16(scales16)); ++ auto m16 = vld1q_f16((const float16_t *)iq8[4*ib4+k].d+8); ++ auto m1 = vcvt_f32_f16(vget_low_f16 (m16)); ++ auto m2 = vcvt_f32_f16(vget_high_f16(m16)); ++ for (int j = 0; j < 16; ++j) qx[j] = vld1q_s8(iq8[4*ib4+k].qs + 16*j); ++ int32x4_t sumi1, sumi2; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ qx_0_q8_0_dot(qx, q8.y[iy][ib4].qs+32*k, sumi1, sumi2); ++ auto dy = vdupq_n_f32(d8[8*iy+k]); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales1, dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales2, dy), vcvtq_f32_s32(sumi2)); ++ auto my = vdupq_n_f32(d8[8*iy+k+4]); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], m1, my); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], m2, my); ++ } ++ } ++ } ++ for (int ib = 4*(nb/4); ib < nb; ++ib) { ++ auto scales16 = vld1q_f16((const float16_t *)iq8[ib].d); ++ auto scales1 = vcvt_f32_f16(vget_low_f16 (scales16)); ++ auto scales2 = vcvt_f32_f16(vget_high_f16(scales16)); ++ auto m16 = vld1q_f16((const float16_t *)iq8[ib].d+8); ++ auto m1 = vcvt_f32_f16(vget_low_f16 (m16)); ++ auto m2 = vcvt_f32_f16(vget_high_f16(m16)); ++ for (int j = 0; j < 16; ++j) qx[j] = vld1q_s8(iq8[ib].qs + 16*j); ++ int32x4_t sumi1, sumi2; ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ auto qy = (const block_q8_1 *)q8.y[iy]; ++ qx_0_q8_0_dot(qx, qy[ib].qs, sumi1, sumi2); ++ auto dy = vdupq_n_f32(GGML_FP16_TO_FP32(qy[ib].d)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], vmulq_f32(scales1, dy), vcvtq_f32_s32(sumi1)); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], vmulq_f32(scales2, dy), vcvtq_f32_s32(sumi2)); ++ auto my = vdupq_n_f32(GGML_FP16_TO_FP32(qy[ib].s)); ++ acc[2*iy+0] = vfmaq_f32(acc[2*iy+0], m1, my); ++ acc[2*iy+1] = vfmaq_f32(acc[2*iy+1], m2, my); ++ } ++ } ++ for (int iy = 0; iy < nrc_y; ++iy) { ++ info.store(ix+0, iy, acc[2*iy+0]); ++ info.store(ix+4, iy, acc[2*iy+1]); ++ acc[2*iy] = acc[2*iy+1] = vdupq_n_f32(0.f); ++ } ++ } ++} ++ ++struct DeqQ40 { ++ const int8x16_t m8 = vdupq_n_s8(-8); ++ const uint8x16_t ml = vdupq_n_s8(0xf); ++ inline int8x16x2_t dequant(const block_q4_0& x) const { ++ auto bits = vld1q_u8(x.qs); ++ return { vaddq_s8(vreinterpretq_s8_u8(vandq_u8(bits, ml)), m8), vaddq_s8(vreinterpretq_s8_u8(vshrq_n_u8(bits, 4)), m8) }; ++ } ++}; ++ ++struct DeqQ41 { ++ const uint8x16_t ml = vdupq_n_s8(0xf); ++ inline int8x16x2_t dequant(const block_q4_1& x) const { ++ auto bits = vld1q_u8(x.qs); ++ return { vreinterpretq_s8_u8(vandq_u8(bits, ml)), vreinterpretq_s8_u8(vshrq_n_u8(bits, 4)) }; ++ } ++}; ++ ++struct DeqIQ4NL { ++ const int8x16_t mt = load_values(); ++ const uint8x16_t ml = vdupq_n_s8(0xf); ++ inline int8x16x2_t dequant(const block_iq4_nl& x) const { ++ auto bits = vld1q_u8(x.qs); ++ return { vqtbl1q_s8(mt, vandq_u8(bits, ml)), vqtbl1q_s8(mt, vshrq_n_u8(bits, 4)) }; ++ } ++ static inline int8x16_t load_values() { return vld1q_s8(iq4k_values); } ++}; ++ ++struct DeqMXFP4 { ++ const int8x16_t mt = load_values(); ++ const uint8x16_t ml = vdupq_n_s8(0xf); ++ inline int8x16x2_t dequant(const block_mxfp4& x) const { ++ auto bits = vld1q_u8(x.qs); ++ return { vqtbl1q_s8(mt, vandq_u8(bits, ml)), vqtbl1q_s8(mt, vshrq_n_u8(bits, 4)) }; ++ } ++ static inline int8x16_t load_values() { return vld1q_s8(kvalues_mxfp4); } ++}; ++ ++struct DeqQ50 { ++ ++ inline int8x16x2_t dequant(const block_q5_0& x) const { ++ int8x16x2_t r; ++ bits.prepare1(x.qs, r.val); ++ auto qh = x.qh; ++ r.val[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(r.val[0]), vandq_u8(mh, hbits.to_negated_bytes(qh+0)))); ++ r.val[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(r.val[1]), vandq_u8(mh, hbits.to_negated_bytes(qh+2)))); ++ return r; ++ } ++ ++ Q4LegacyBits bits; ++ HighBit5Legacy hbits; ++ const uint8x16_t mh = vdupq_n_u8(0xf0); ++}; ++ ++struct DeqQ51 { ++ ++ inline int8x16x2_t dequant(const block_q5_1& x) const { ++ int8x16x2_t r; ++ bits.prepare1(x.qs, r.val); ++ auto qh = x.qh; ++ r.val[0] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(r.val[0]), vandq_u8(mh, hbits.to_bytes(qh+0)))); ++ r.val[1] = vreinterpretq_s8_u8(vorrq_u8(vreinterpretq_u8_s8(r.val[1]), vandq_u8(mh, hbits.to_bytes(qh+2)))); ++ return r; ++ } ++ ++ Q4LegacyBits bits; ++ HighBit5Legacy hbits; ++ const uint8x16_t mh = vdupq_n_u8(0x10); ++}; ++ ++struct DeqQ60 { ++ ++ inline int8x16x2_t dequant(const block_q6_0& x) const { ++ int8x16x2_t r; ++ bits.prepare1(x.qs, r.val); ++ auto qh8 = vld1_u8(x.qh); ++ auto qh = vcombine_u8(vshl_n_u8(qh8, 4), qh8); ++ r.val[0] = vaddq_s8(vorrq_u8(r.val[0], vandq_u8(qh, hmask)), m32); ++ r.val[1] = vaddq_s8(vorrq_u8(r.val[1], vandq_u8(vshrq_n_u8(qh, 2), hmask)), m32); ++ return r; ++ } ++ ++ Q4LegacyBits bits; ++ const int8x16_t m32 = vdupq_n_s8(-32); ++ const uint8x16_t hmask = vdupq_n_u8(0x30); ++}; ++ ++struct DeqQ80 { ++ inline int8x16x2_t dequant(const block_q8_0& x) const { ++ return vld1q_s8_x2(x.qs); ++ } ++}; ++ ++template ++void iqk_convert_qX_q80_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK4_0 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ const int nb = n/QK8_0; ++ ++ block_q8_0_r8 * y = (block_q8_0_r8 *)vy; ++ ++ const Block * x8[8]; ++ ++ uint32_t block[8]; ++ ++ Dequantizer deq; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ ++ for (int k = 0; k < 8; ++k) x8[k] = (const Block *)((const char *)vx + (ix + k)*bx); ++ ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ if constexpr (std::is_same_v) { ++ y[i].d[k] = GGML_FP32_TO_FP16(GGML_E8M0_TO_FP32_HALF(x8[k][i].e)); ++ } else { ++ y[i].d[k] = x8[k][i].d; ++ } ++ vst1q_s8_x2((int8_t *)block, deq.dequant(x8[k][i])); ++ auto qs = (uint32_t *)y[i].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++template ++void iqk_convert_qX_1_q8_1_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ GGML_ASSERT(n%QK4_0 == 0); ++ GGML_ASSERT(nrc_x%8 == 0); ++ ++ const int nb = n/QK8_0; ++ ++ block_q8_1_r8 * y = (block_q8_1_r8 *)vy; ++ ++ const Block * x8[8]; ++ ++ uint32_t block[8]; ++ ++ Dequantizer deq; ++ ++ for (int ix = 0; ix < nrc_x; ix += 8) { ++ ++ for (int k = 0; k < 8; ++k) x8[k] = (const Block *)((const char *)vx + (ix + k)*bx); ++ ++ for (int i = 0; i < nb; ++i) { ++ for (int k = 0; k < 8; ++k) { ++ y[i].d[k+0] = x8[k][i].d; ++ y[i].d[k+8] = x8[k][i].m; ++ vst1q_s8_x2((int8_t *)block, deq.dequant(x8[k][i])); ++ auto qs = (uint32_t *)y[i].qs; ++ for (int l = 0; l < 4; ++l) { ++ qs[8*l + k + 0] = block[l + 0]; ++ qs[8*l + k + 32] = block[l + 4]; ++ } ++ } ++ } ++ y += nb; ++ } ++} ++ ++} ++ ++bool iqk_convert_legacy_quants_q8_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x) { ++ switch (type) { ++ case GGML_TYPE_Q4_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q4_1 : iqk_convert_qX_1_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q5_1 : iqk_convert_qX_1_q8_1_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q6_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_IQ4_NL: iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_MXFP4 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ case GGML_TYPE_Q8_0 : iqk_convert_qX_q80_r8(n, vx, bx, vy, nrc_x); break; ++ default: return false; ++ } ++ return true; ++} ++ ++bool iqk_set_kernels_legacy_quants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16) { ++ ++ if (ne00%QK8_0 != 0) return false; ++ ++ auto etypeA = ggml_type(typeA); ++ auto expected_typeB = etypeA == GGML_TYPE_Q4_1 || etypeA == GGML_TYPE_Q5_1 || etypeA == GGML_TYPE_Q8_1 ? GGML_TYPE_Q8_1_X4 : GGML_TYPE_Q8_0_X4; ++ if (ggml_type(typeB) != expected_typeB) return false; ++ ++ func16 = nullptr; ++ ++ switch (typeA) { ++ case GGML_TYPE_Q4_0: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerQ40, kernels); ++ break; ++ case GGML_TYPE_Q4_1: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_1_q8_1, DequantizerQ41, kernels); ++ break; ++ case GGML_TYPE_Q5_0: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerQ50, kernels); ++ break; ++ case GGML_TYPE_Q5_1: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_1_q8_1, DequantizerQ51, kernels); ++ break; ++ case GGML_TYPE_Q6_0: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerQ60, kernels); ++ break; ++ case GGML_TYPE_Q8_0: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerQ80, kernels); ++ break; ++ case GGML_TYPE_IQ4_NL: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerIQ4NL, kernels); ++ break; ++ case GGML_TYPE_MXFP4: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qX_0_q8_0, DequantizerMXFP4, kernels); ++ break; ++ case GGML_TYPE_Q4_0_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qx_r8_q8_0, Q4_0_R8_Dequantizer, kernels); ++ break; ++ case GGML_TYPE_Q5_0_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qx_r4_q8_0, Q5_0_R4_Dequantizer, kernels); ++ break; ++ case GGML_TYPE_Q6_0_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qx_r4_q8_0, Q6_0_R4_Dequantizer, kernels); ++ break; ++ case GGML_TYPE_Q8_0_R8: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_0_r8_q8_0, kernels); ++ break; ++ case GGML_TYPE_Q8_1: ++ IQK_SET_MUL_MAT_FUNCTIONS(mul_mat_q8_1_r8_q8_1, kernels); ++ break; ++ case GGML_TYPE_IQ4_NL_R4: ++ IQK_SET_MUL_MAT_FUNCTIONS_T(mul_mat_qx_r4_q8_0, IQ4_NL_R4_Dequantizer, kernels); ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++} ++ ++#endif ++ ++namespace { ++template ++inline std::pair mul_mat_kernel(int int_typeA, int nq) { ++ auto typeA = ggml_type(int_typeA); ++ constexpr int kMaxQ = 8; ++#define MAKE_FUNCS(mul_mat, n) \ ++ if (n >= kMaxQ) return std::make_pair(mul_mat, kMaxQ>, kMaxQ);\ ++ else {\ ++ switch (n) {\ ++ case 1: return std::make_pair(mul_mat, 1>, 1);\ ++ case 2: return std::make_pair(mul_mat, 2>, 2);\ ++ case 3: return std::make_pair(mul_mat, 3>, 3);\ ++ case 4: return std::make_pair(mul_mat, 4>, 4);\ ++ case 5: return std::make_pair(mul_mat, 5>, 5);\ ++ case 6: return std::make_pair(mul_mat, 6>, 6);\ ++ case 7: return std::make_pair(mul_mat, 7>, 7);\ ++ }\ ++ } ++#define MAKE_FUNCS2(mul_mat, block, n) \ ++ if (n >= kMaxQ) return std::make_pair(mul_mat, kMaxQ, block>, kMaxQ);\ ++ else {\ ++ switch (n) {\ ++ case 1: return std::make_pair(mul_mat, 1, block>, 1);\ ++ case 2: return std::make_pair(mul_mat, 2, block>, 2);\ ++ case 3: return std::make_pair(mul_mat, 3, block>, 3);\ ++ case 4: return std::make_pair(mul_mat, 4, block>, 4);\ ++ case 5: return std::make_pair(mul_mat, 5, block>, 5);\ ++ case 6: return std::make_pair(mul_mat, 6, block>, 6);\ ++ case 7: return std::make_pair(mul_mat, 7, block>, 7);\ ++ }\ ++ } ++#define MAKE_FUNCS_ONLY_NRC(mul_mat, n) \ ++ if (n >= kMaxQ) return std::make_pair(mul_mat, kMaxQ);\ ++ else {\ ++ switch (n) {\ ++ case 1: return std::make_pair(mul_mat<1>, 1);\ ++ case 2: return std::make_pair(mul_mat<2>, 2);\ ++ case 3: return std::make_pair(mul_mat<3>, 3);\ ++ case 4: return std::make_pair(mul_mat<4>, 4);\ ++ case 5: return std::make_pair(mul_mat<5>, 5);\ ++ case 6: return std::make_pair(mul_mat<6>, 6);\ ++ case 7: return std::make_pair(mul_mat<7>, 7);\ ++ }\ ++ } ++ if (typeA == GGML_TYPE_Q8_0) { ++#ifdef __aarch64__ ++ MAKE_FUNCS(mul_mat_qX_0_q8_0, 1); ++ if (nq == 2) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 2); ++ if (nq == 4) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 4); ++ MAKE_FUNCS(mul_mat_qX_1_q8_2_T, 1); ++ //if (nq == 2) return std::make_pair(mul_mat_qX_0_q8_0_Tx, 2); ++ //if (nq == 4) return std::make_pair(mul_mat_qX_0_q8_0_Tx, 4); ++ if (nq == 1) return std::make_pair(mul_mat_qX_0_q8_0_T, 1); ++ if (nq == 2) return std::make_pair(mul_mat_qX_0_q8_0_T, 2); ++ if (nq == 4) return std::make_pair(mul_mat_qX_0_q8_0_T, 4); ++ if (nq == 3) return std::make_pair(mul_mat_qX_0_q8_0_T, 3); ++ if (nq == 5) return std::make_pair(mul_mat_qX_0_q8_0_T, 5); ++ if (nq == 6) return std::make_pair(mul_mat_qX_0_q8_0_T, 6); ++ if (nq == 7) return std::make_pair(mul_mat_qX_0_q8_0_T, 7); ++ return std::make_pair(mul_mat_qX_0_q8_0_T, kMaxQ); ++#endif ++#endif ++ } ++ else if (typeA == GGML_TYPE_Q8_0_R8) { ++#ifdef __aarch64__ ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_0_r8_q8_0, nq); ++#else ++ MAKE_FUNCS_ONLY_NRC(mul_mat_q8_0_r8_q8_2, nq); ++#endif ++ } ++ else if (typeA == GGML_TYPE_Q6_0) { ++#ifdef __aarch64__ ++ MAKE_FUNCS(mul_mat_qX_0_q8_0, 1); ++ if (nq == 2) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 2); ++ if (nq == 4) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 4); ++ MAKE_FUNCS(mul_mat_qX_1_q8_2_T, 1); ++ if (nq == 2) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 2); ++ if (nq == 4) return std::make_pair(mul_mat_qX_0_q8_2_Tx, 4); ++ MAKE_FUNCS(mul_mat_qX_1_q8_2_T(nullptr, 0); ++} ++ ++inline std::pair mul_mat_kernel(int int_typeA, int nq, int k_step) { ++ switch (k_step) { ++ case 32: return mul_mat_kernel< 32>(int_typeA, nq); ++ case 64: return mul_mat_kernel< 64>(int_typeA, nq); ++ case 128: return mul_mat_kernel<128>(int_typeA, nq); ++ default: GGML_ABORT("Fatal error"); ++ } ++} ++} ++ ++void iqk_gemm_legacy_fa(int D, int nq, int type_k, const char * k, size_t stride_k, DataInfo& info, int k_step) { ++ auto [mul_mat, nrc_q] = mul_mat_kernel(type_k, nq, k_step); ++ for (int iq = 0; iq < nq/nrc_q; ++iq) { ++ mul_mat(D, k, stride_k, info, k_step); ++ info.cur_y += nrc_q; ++ } ++ int iq = nrc_q*(nq/nrc_q); ++ if (iq < nq) { ++ auto [mul_mat1, nrc_q1] = mul_mat_kernel(type_k, nq - iq, k_step); ++ GGML_ASSERT(nrc_q1 == nq - iq); ++ mul_mat1(D, k, stride_k, info, k_step); ++ } ++} ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.h b/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_gemm_legacy_quants.h +@@ -0,0 +1,16 @@ ++#pragma once ++ ++#include "iqk_common.h" ++ ++#ifdef IQK_IMPLEMENT ++ ++#include ++#include ++ ++bool iqk_set_kernels_legacy_quants(int ne00, int typeA, int typeB, std::array& kernels, mul_mat_t& func16); ++ ++void iqk_gemm_legacy_fa(int D, int nq, int type_k, const char * k, size_t stride_k, DataInfo& info, int k_step); ++ ++bool iqk_convert_legacy_quants_q8_r8(int type, int n, const void * vx, size_t bx, void * vy, int nrc_x); ++ ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h b/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h +@@ -0,0 +1,192 @@ ++// opencoti F5-opt W2 (#290) — ik_llama ggml_type enum DELTA. ++// IK-only enumerators (names absent from llamafile's ggml.h), as ++// ((ggml_type)N) macros so iqk code can switch/compare on them. Values ++// are >41 (llamafile's max), so they never collide with llamafile's enum. ++// VENDORED from ik_llama.cpp @ 8960c5ba ggml/include/ggml.h. MIT, (C) 2024 IK. ++#ifndef IQK_GGML_TYPE_EXT_H ++#define IQK_GGML_TYPE_EXT_H ++#include "ggml.h" ++#ifndef GGML_TYPE_Q4_0_4_4 ++#define GGML_TYPE_Q4_0_4_4 ((ggml_type)31) ++#endif ++#ifndef GGML_TYPE_Q4_0_4_8 ++#define GGML_TYPE_Q4_0_4_8 ((ggml_type)32) ++#endif ++#ifndef GGML_TYPE_Q4_0_8_8 ++#define GGML_TYPE_Q4_0_8_8 ((ggml_type)33) ++#endif ++#ifndef GGML_TYPE_I2_S ++#define GGML_TYPE_I2_S ((ggml_type)36) ++#endif ++#ifndef GGML_TYPE_Q1_0_G128 ++#define GGML_TYPE_Q1_0_G128 ((ggml_type)41) ++#endif ++#ifndef GGML_TYPE_Q8_0_X4 ++#define GGML_TYPE_Q8_0_X4 ((ggml_type)97) ++#endif ++#ifndef GGML_TYPE_Q8_1_X4 ++#define GGML_TYPE_Q8_1_X4 ((ggml_type)98) ++#endif ++#ifndef GGML_TYPE_Q8_2_X4 ++#define GGML_TYPE_Q8_2_X4 ((ggml_type)99) ++#endif ++#ifndef GGML_TYPE_Q6_0 ++#define GGML_TYPE_Q6_0 ((ggml_type)133) ++#endif ++#ifndef GGML_TYPE_IQ1_BN ++#define GGML_TYPE_IQ1_BN ((ggml_type)134) ++#endif ++#ifndef GGML_TYPE_IQ2_BN ++#define GGML_TYPE_IQ2_BN ((ggml_type)135) ++#endif ++#ifndef GGML_TYPE_Q8_K64 ++#define GGML_TYPE_Q8_K64 ((ggml_type)136) ++#endif ++#ifndef GGML_TYPE_IQ2_K ++#define GGML_TYPE_IQ2_K ((ggml_type)137) ++#endif ++#ifndef GGML_TYPE_IQ3_K ++#define GGML_TYPE_IQ3_K ((ggml_type)138) ++#endif ++#ifndef GGML_TYPE_IQ4_K ++#define GGML_TYPE_IQ4_K ((ggml_type)139) ++#endif ++#ifndef GGML_TYPE_IQ5_K ++#define GGML_TYPE_IQ5_K ((ggml_type)140) ++#endif ++#ifndef GGML_TYPE_IQ6_K ++#define GGML_TYPE_IQ6_K ((ggml_type)141) ++#endif ++#ifndef GGML_TYPE_IQ4_KS ++#define GGML_TYPE_IQ4_KS ((ggml_type)144) ++#endif ++#ifndef GGML_TYPE_IQ2_KS ++#define GGML_TYPE_IQ2_KS ((ggml_type)145) ++#endif ++#ifndef GGML_TYPE_IQ4_KSS ++#define GGML_TYPE_IQ4_KSS ((ggml_type)146) ++#endif ++#ifndef GGML_TYPE_Q8_K16 ++#define GGML_TYPE_Q8_K16 ((ggml_type)147) ++#endif ++#ifndef GGML_TYPE_Q8_K32 ++#define GGML_TYPE_Q8_K32 ((ggml_type)148) ++#endif ++#ifndef GGML_TYPE_Q8_KR8 ++#define GGML_TYPE_Q8_KR8 ((ggml_type)149) ++#endif ++#ifndef GGML_TYPE_Q8_K128 ++#define GGML_TYPE_Q8_K128 ((ggml_type)150) ++#endif ++#ifndef GGML_TYPE_Q8_KV ++#define GGML_TYPE_Q8_KV ((ggml_type)151) ++#endif ++#ifndef GGML_TYPE_IQ5_KS ++#define GGML_TYPE_IQ5_KS ((ggml_type)152) ++#endif ++#ifndef GGML_TYPE_IQ2_KT ++#define GGML_TYPE_IQ2_KT ((ggml_type)153) ++#endif ++#ifndef GGML_TYPE_IQ3_KT ++#define GGML_TYPE_IQ3_KT ((ggml_type)154) ++#endif ++#ifndef GGML_TYPE_IQ4_KT ++#define GGML_TYPE_IQ4_KT ((ggml_type)155) ++#endif ++#ifndef GGML_TYPE_IQ3_KS ++#define GGML_TYPE_IQ3_KS ((ggml_type)156) ++#endif ++#ifndef GGML_TYPE_IQ2_KL ++#define GGML_TYPE_IQ2_KL ((ggml_type)157) ++#endif ++#ifndef GGML_TYPE_IQ1_KT ++#define GGML_TYPE_IQ1_KT ((ggml_type)158) ++#endif ++#ifndef GGML_TYPE_Q4_0_R8 ++#define GGML_TYPE_Q4_0_R8 ((ggml_type)202) ++#endif ++#ifndef GGML_TYPE_Q5_0_R4 ++#define GGML_TYPE_Q5_0_R4 ((ggml_type)206) ++#endif ++#ifndef GGML_TYPE_Q8_0_R8 ++#define GGML_TYPE_Q8_0_R8 ((ggml_type)208) ++#endif ++#ifndef GGML_TYPE_Q2_K_R4 ++#define GGML_TYPE_Q2_K_R4 ((ggml_type)210) ++#endif ++#ifndef GGML_TYPE_Q3_K_R4 ++#define GGML_TYPE_Q3_K_R4 ((ggml_type)211) ++#endif ++#ifndef GGML_TYPE_Q4_K_R4 ++#define GGML_TYPE_Q4_K_R4 ((ggml_type)212) ++#endif ++#ifndef GGML_TYPE_Q5_K_R4 ++#define GGML_TYPE_Q5_K_R4 ((ggml_type)213) ++#endif ++#ifndef GGML_TYPE_Q6_K_R4 ++#define GGML_TYPE_Q6_K_R4 ((ggml_type)214) ++#endif ++#ifndef GGML_TYPE_IQ2_XXS_R4 ++#define GGML_TYPE_IQ2_XXS_R4 ((ggml_type)216) ++#endif ++#ifndef GGML_TYPE_IQ2_XS_R4 ++#define GGML_TYPE_IQ2_XS_R4 ((ggml_type)217) ++#endif ++#ifndef GGML_TYPE_IQ3_XXS_R4 ++#define GGML_TYPE_IQ3_XXS_R4 ((ggml_type)218) ++#endif ++#ifndef GGML_TYPE_IQ1_S_R4 ++#define GGML_TYPE_IQ1_S_R4 ((ggml_type)219) ++#endif ++#ifndef GGML_TYPE_IQ4_NL_R4 ++#define GGML_TYPE_IQ4_NL_R4 ((ggml_type)220) ++#endif ++#ifndef GGML_TYPE_IQ3_S_R4 ++#define GGML_TYPE_IQ3_S_R4 ((ggml_type)221) ++#endif ++#ifndef GGML_TYPE_IQ2_S_R4 ++#define GGML_TYPE_IQ2_S_R4 ((ggml_type)222) ++#endif ++#ifndef GGML_TYPE_IQ4_XS_R8 ++#define GGML_TYPE_IQ4_XS_R8 ((ggml_type)223) ++#endif ++#ifndef GGML_TYPE_IQ1_M_R4 ++#define GGML_TYPE_IQ1_M_R4 ((ggml_type)229) ++#endif ++#ifndef GGML_TYPE_BF16_R16 ++#define GGML_TYPE_BF16_R16 ((ggml_type)230) ++#endif ++#ifndef GGML_TYPE_Q6_0_R4 ++#define GGML_TYPE_Q6_0_R4 ((ggml_type)233) ++#endif ++#ifndef GGML_TYPE_IQ2_BN_R4 ++#define GGML_TYPE_IQ2_BN_R4 ((ggml_type)335) ++#endif ++#ifndef GGML_TYPE_IQ2_K_R4 ++#define GGML_TYPE_IQ2_K_R4 ((ggml_type)337) ++#endif ++#ifndef GGML_TYPE_IQ3_K_R4 ++#define GGML_TYPE_IQ3_K_R4 ((ggml_type)338) ++#endif ++#ifndef GGML_TYPE_IQ4_K_R4 ++#define GGML_TYPE_IQ4_K_R4 ((ggml_type)339) ++#endif ++#ifndef GGML_TYPE_IQ5_K_R4 ++#define GGML_TYPE_IQ5_K_R4 ((ggml_type)340) ++#endif ++#ifndef GGML_TYPE_IQ4_KS_R4 ++#define GGML_TYPE_IQ4_KS_R4 ((ggml_type)344) ++#endif ++#ifndef GGML_TYPE_IQ5_KS_R4 ++#define GGML_TYPE_IQ5_KS_R4 ((ggml_type)352) ++#endif ++#ifndef GGML_TYPE_Q8_K_R16 ++#define GGML_TYPE_Q8_K_R16 ((ggml_type)397) ++#endif ++#ifndef GGML_TYPE_Q8_KV_R8 ++#define GGML_TYPE_Q8_KV_R8 ((ggml_type)398) ++#endif ++#ifndef GGML_TYPE_Q8_K_R8 ++#define GGML_TYPE_Q8_K_R8 ((ggml_type)399) ++#endif ++#endif // IQK_GGML_TYPE_EXT_H +diff --git a/llama.cpp/ggml/src/iqk/iqk_mul_mat.h b/llama.cpp/ggml/src/iqk/iqk_mul_mat.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_mul_mat.h +@@ -0,0 +1,83 @@ ++// ++// Copyright (C) 2024 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++#include ++#include ++#include ++#include "iqk_config.h" ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++IQK_API bool iqk_mul_mat(long Nx, long Ny, long ne00, ++ int typeA, const void * A, long strideA, ++ int typeB, const void * B, long strideB, ++ float * C, long stride_C, int ith, int nth); ++ ++IQK_API bool iqk_mul_mat_4d(long Nx, long Ny, long ne00, ++ long ne02, long ne03, long ne12, long ne13, ++ long nb02, long nb03, long nb12, long nb13, long nb2, long nb3, ++ int typeA, const void * A, long strideA, ++ int typeB, const void * B, long strideB, ++ float * C, long stride_C, int ith, int nth); ++ ++IQK_API bool iqk_mul_mat_moe(long Nx, long Ny, long ne00, int ne11, ++ int typeA, const void * A, long strideA, ++ int typeB, const void * B, long strideB, ++ float * C, long nb1, long nb2, const void * vrow_mapping, int ith, int nth); ++ ++IQK_API bool iqk_moe_fused_up_gate(long Nx, long Ny, long ne00, int ne11, int unary_op, ++ int typeA, const void * Aup, const void * Agate, long strideA, ++ int typeB, const void * B, long strideB, ++ const char * up_b, const char * gate_b, ++ float * C, long nb1, long nb2, const void * vrow_mapping, float limit, int ith, int nth); ++ ++IQK_API int iqk_dequant_type(int type, int Ny); ++ ++struct ggml_tensor; ++ ++IQK_API size_t iqk_fa_work_buffer_size(const struct ggml_tensor * dst, int nthread); ++ ++typedef void (*barrier_t) (void *); ++ ++IQK_API bool iqk_flash_attn_noalibi(int type_q, int type_mask, float max_bias, ++ int neq3, int neq2, long nbq3, long nbq2, ++ int nek3, int nek2, long nbk3, long nbk2, ++ int nev3, int nev2, long nbv3, long nbv2, ++ int ne2, int ne1, long nb1, ++ int type_k, // type of k ++ int type_v, // type of v ++ int Dk, // K head size ++ int Dv, // V head size ++ int nq, // number of columns in q ++ int nk, // number of rows in k ++ int stride_q, // distance between q columns in bytes ++ int stride_k, // distance between k rows in bytes ++ int stride_v, // distance between v rows in bytes ++ int stride_m, // distance between mask rows (in bytes ++ const void * q, // q matrix. ++ const void * k, // k matrix. Assumed to be fp16, nq x nk elements ++ const void * v, // v matrix. Assumed to be fp16, nq x nk elements ++ const void * mask, // mask. If not null, assumed to be fp16. nq x nk elements ++ const void * sinks, // mask. If not null, assumed to be fp16. nq x nk elements ++ float scale, // scale applied before softmax ++ float softcap, // if > 0, a "soft-cap" operation is applied before softmax ++ float * qkv, // v*softmax(scale*(k*q)) ++ void * work_buffer, barrier_t barrier, void * barrier_data, ++ int ith, int nth, int n_swa); ++ ++IQK_API void iqk_topk_moe(int n_experts, int n_experts_used, int nrows, const float * logits, ++ float * weights, int32_t * ids, int ith, int nth); ++ ++IQK_API bool iqk_fused_delta_net(int head_dim, int n_heads, int gqa_ratio, int repeat_type, int n_tokens, int n_seqs, ++ size_t vnb1, size_t vnb2, size_t vnb3, ++ const float * q_data, const float * k_data, const float * v_data, const float * g_data, const float * beta_data, ++ const float * state_in, float * out_data, float * state_out, float * saved_steps, int state_step_stride, int ith, int nth); ++ ++#ifdef __cplusplus ++} ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_quantize.cpp b/llama.cpp/ggml/src/iqk/iqk_quantize.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_quantize.cpp +@@ -0,0 +1,10497 @@ ++// ++// Copyright (C) 2024 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#if GGML_USE_IQK_MULMAT ++#include "iqk_mul_mat.h" ++#endif ++#include "ggml-quants.h" ++#include "ggml-impl.h" ++#define GGML_COMMON_IMPL_C ++#include "ggml-common.h" ++#include "iqk_quantize.h" ++#include "iqk_config.h" ++ ++#include "iqk_gemm_ktquants.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++inline int nearest_int(float fval) { ++ assert(fval <= 4194303.f); ++ float val = fval + 12582912.f; ++ int i; memcpy(&i, &val, sizeof(int)); ++ return (i & 0x007fffff) - 0x00400000; ++} ++ ++typedef void (*quantize_func_t)(const float * src, void * qdata, int n_per_row, const float * imatrix, const quantize_user_data * user_data); ++ ++struct QHelper { ++ QHelper(const float * imatrix, const quantize_user_data * user_data, int n_per_row, int block_size) : ++ m_imatrix(imatrix), m_user_data(user_data), m_n_per_row(n_per_row), m_block_size(block_size) { ++ if (m_imatrix) { ++ m_weight.resize(m_n_per_row); ++ } ++ } ++ const float * row_weights(const float * x) { ++ constexpr float kEps = 1e-9f; ++ constexpr float kEps2 = kEps*kEps; ++ if (!m_imatrix) return m_imatrix; ++ int nblock = m_n_per_row / m_block_size; ++ for (int ib = 0; ib < nblock; ++ib) { ++ auto wb_in = m_imatrix + ib*m_block_size; ++ auto xb = x + ib*m_block_size; ++ auto wb = m_weight.data() + ib*m_block_size; ++ float sumw2 = 0, sumx2 = 0, sumwx = 0; ++ for (int j = 0; j < m_block_size; ++j) { ++ wb[j] = wb_in[j]; ++ sumw2 += wb[j]*wb[j]; ++ sumx2 += xb[j]*xb[j]; ++ sumwx += wb[j]*std::abs(xb[j]); ++ } ++ if (sumw2 > m_block_size*kEps2 && sumx2 > m_block_size*kEps2 && sumwx > m_block_size*kEps2) continue; ++ for (int j = 0; j < m_block_size; ++j) { ++ wb[j] = kEps; ++ } ++ } ++ return m_weight.data(); ++ } ++ template ++ void quantize(int nrows, const float * src, void * dst, int row_size, const Func& qfunc) { ++ auto cdst = (char *)dst; ++ for (int row = 0; row < nrows; ++row) { ++ auto weights = row_weights(src); ++ qfunc(src, cdst, m_n_per_row, weights, m_user_data); ++ src += m_n_per_row; ++ cdst += row_size; ++ } ++ } ++private: ++ const float * m_imatrix; ++ const quantize_user_data * m_user_data; ++ const int m_n_per_row; ++ const int m_block_size; ++ std::vector m_weight; ++}; ++ ++template ++size_t quantize_repack(ggml_type type, const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data, const Func& q_func, const RepackFunc& repack) { ++ GGML_ASSERT(nrows%n_repack == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(type, n_per_row); ++ std::vector qtmp(n_repack*row_size); ++ QHelper helper(imatrix, user_data, n_per_row, block_size); ++ char * qrow = (char *)dst; ++ for (int row = 0; row < nrows; row += n_repack) { ++ helper.quantize(n_repack, src, qtmp.data(), row_size, q_func); ++ repack(n_repack, n_per_row, (const Block *)qtmp.data(), (Block_repacked *)qrow, false); ++ src += n_repack*n_per_row; ++ qrow += n_repack*row_size; ++ } ++ return nrows*row_size; ++} ++ ++ ++float make_qx_quants(int n, int nmax, const float * x, int8_t * L, const float * qw) { ++ float max = 0; ++ float amax = 0; ++ for (int i = 0; i < n; ++i) { ++ float ax = fabsf(x[i]); ++ if (ax > amax) { amax = ax; max = x[i]; } ++ } ++ if (!amax) { // all zero ++ for (int i = 0; i < n; ++i) L[i] = 0; ++ return 0.f; ++ } ++ float iscale = -nmax / max; ++ float sumlx = 0; ++ float suml2 = 0; ++ for (int i = 0; i < n; ++i) { ++ int l = nearest_int(iscale * x[i]); ++ l = std::max(-nmax, std::min(nmax-1, l)); ++ L[i] = l + nmax; ++ sumlx += qw[i]*x[i]*l; ++ suml2 += qw[i]*l*l; ++ } ++ float scale = suml2 ? sumlx/suml2 : 0.0f; ++ float best = scale * sumlx; ++ for (int is = -9; is <= 9; ++is) { ++ if (is == 0) continue; ++ iscale = -(nmax + 0.1f*is) / max; ++ sumlx = suml2 = 0; ++ for (int i = 0; i < n; ++i) { ++ int l = nearest_int(iscale * x[i]); ++ l = std::max(-nmax, std::min(nmax-1, l)); ++ sumlx += qw[i]*x[i]*l; ++ suml2 += qw[i]*l*l; ++ } ++ if (suml2 > 0 && sumlx*sumlx > best*suml2) { ++ for (int i = 0; i < n; ++i) { ++ int l = nearest_int(iscale * x[i]); ++ L[i] = nmax + std::max(-nmax, std::min(nmax-1, l)); ++ } ++ scale = sumlx/suml2; best = scale*sumlx; ++ } ++ } ++ return scale; ++} ++ ++struct IQ1BNQuantizer { ++ int8_t L[QK_IQ1BN]; ++ void quantize_one_row_1bn(const float * src, block_iq1_bn * y, int n_per_row, const float * imatrix); ++ void quantize_one_row_2bn(const float * src, block_iq2_bn * y, int n_per_row, const float * imatrix); ++ static inline float row_max(int n_per_row, const float * src) { ++ float max_in_row = 0; ++ for (int j = 0; j < n_per_row; ++j) { ++ float ax = fabsf(src[j]); ++ max_in_row = std::max(max_in_row, ax); ++ } ++ return max_in_row; ++ } ++ // The Makefile has issues dwaling with this? ++ //static constexpr uint8_t k_mult[5] = {81, 27, 9, 3, 1}; ++ static const uint8_t k_mult[5]; ++}; ++ ++const uint8_t IQ1BNQuantizer::k_mult[5] = {81, 27, 9, 3, 1}; ++ ++void IQ1BNQuantizer::quantize_one_row_1bn(const float * src, block_iq1_bn * y, int n_per_row, const float * imatrix) { ++ ++ static const int k_nb[6] = {1, 3, 9, 27, 81, 243}; ++ (void)imatrix; ++ ++ const int nblock = n_per_row/QK_IQ1BN; ++ ++ ggml_half * dptr = (ggml_half *)y; ++ y = (block_iq1_bn *)(dptr + 1); ++ ++ float max = 0; ++ for (int j = 0; j < n_per_row; ++j) max = std::max(max, fabsf(src[j])); ++ ggml_half d = GGML_FP32_TO_FP16(max); ++ std::memcpy(dptr, &d, sizeof(d)); ++ ++ float thresh = 0.5f*max; ++ ++ for (int ib = 0; ib < nblock; ++ib) { ++ std::memset(&y[ib], 0, sizeof(block_iq1_bn)); ++ auto xb = src + ib*QK_IQ1BN; ++ int v13 = 0; ++ for (int i16 = 0; i16 < QK_IQ1BN/16; ++i16) { ++ for (int k = 0; k < 3; ++k) { ++ int idx = 0; ++ for (int j = 0; j < 5; ++j) { ++ float v = xb[16*i16 + 5*k + j]; ++ int q = fabsf(v) < thresh ? 1 : v < 0 ? 0 : 2; ++ idx += k_nb[j]*q; ++ } ++ idx = (256*idx + k_nb[5] - 1)/k_nb[5]; ++ y[ib].ql[3*i16 + k] = idx; ++ } ++ float v = xb[16*i16 + 15]; ++ int q = fabsf(v) < thresh ? 1 : v < 0 ? 0 : 2; ++ v13 += k_nb[i16]*q; ++ } ++ y[ib].extra = (256*v13 + k_nb[5] - 1)/k_nb[5]; ++ } ++} ++ ++void IQ1BNQuantizer::quantize_one_row_2bn(const float * src, block_iq2_bn * y, int n_per_row, const float * imatrix) { ++ ++ (void)imatrix; ++ ++ const int nblock = n_per_row/QK_IQ1BN; ++ ++ constexpr int Nj = QK_IQ1BN/4; ++ ++ float max = 0; ++ for (int j = 0; j < n_per_row; ++j) max = std::max(max, fabsf(src[j])); ++ ++ float * dptr = (float *)y; ++ *dptr = max; ++ y = (block_iq2_bn *)(dptr + 1); ++ float thresh = 0.5f*max; ++ ++ for (int ib = 0; ib < nblock; ++ib) { ++ auto xb = src + QK_IQ1BN*ib; ++ for (int j = 0; j < QK_IQ1BN; ++j) { ++ L[j] = fabsf(xb[j]) < thresh ? 1 : xb[j] < 0 ? 0 : 2; ++ } ++ for (int j = 0; j < Nj; ++j) { ++ y[ib].qs[j] = L[j] | (L[j + Nj] << 2) | (L[j + 2*Nj] << 4) | (L[j + 3*Nj] << 6); ++ } ++ } ++} ++ ++static inline int num_rows([[maybe_unused]] ggml_type type) { ++#ifdef HAVE_FANCY_SIMD ++ switch (type) { ++ case GGML_TYPE_Q2_K_R4: ++ case GGML_TYPE_Q3_K_R4: ++ case GGML_TYPE_Q6_K_R4: ++ case GGML_TYPE_IQ2_K_R4: ++ case GGML_TYPE_IQ3_K_R4: ++ case GGML_TYPE_IQ4_K_R4: ++ case GGML_TYPE_IQ5_K_R4: ++ case GGML_TYPE_IQ4_KS_R4: ++ case GGML_TYPE_IQ5_KS_R4: ++ case GGML_TYPE_IQ2_XXS_R4: ++ case GGML_TYPE_IQ2_XS_R4: ++ case GGML_TYPE_IQ2_S_R4: ++ case GGML_TYPE_IQ3_XXS_R4: ++ case GGML_TYPE_IQ1_S_R4: ++ case GGML_TYPE_IQ1_M_R4: ++ case GGML_TYPE_IQ3_S_R4: return 4; ++ case GGML_TYPE_IQ4_NL_R4: ++ case GGML_TYPE_Q5_0_R4: ++ case GGML_TYPE_Q6_0_R4: ++ case GGML_TYPE_IQ2_BN_R4: ++ case GGML_TYPE_IQ4_XS_R8: ++ case GGML_TYPE_Q4_K_R4: ++ case GGML_TYPE_Q5_K_R4: ++ case GGML_TYPE_Q8_KV: ++ case GGML_TYPE_Q8_KV_R8: ++ case GGML_TYPE_Q8_K_R8: return 8; ++ case GGML_TYPE_Q4_0_R8: ++ case GGML_TYPE_Q8_0_R8: ++ case GGML_TYPE_Q8_1: ++ case GGML_TYPE_Q8_K_R16: ++ case GGML_TYPE_BF16_R16: return 16; ++ default: return 1; ++ } ++#else ++ switch (type) { ++ case GGML_TYPE_Q2_K_R4: ++ case GGML_TYPE_Q3_K_R4: ++ case GGML_TYPE_Q4_K_R4: ++ case GGML_TYPE_Q5_K_R4: ++ case GGML_TYPE_Q6_K_R4: ++ case GGML_TYPE_Q5_0_R4: ++ case GGML_TYPE_Q6_0_R4: ++ case GGML_TYPE_IQ4_NL_R4: ++ case GGML_TYPE_IQ2_K_R4: ++ case GGML_TYPE_IQ3_K_R4: ++ case GGML_TYPE_IQ4_K_R4: ++ case GGML_TYPE_IQ5_K_R4: ++ case GGML_TYPE_IQ4_KS_R4: ++ case GGML_TYPE_IQ5_KS_R4: ++ case GGML_TYPE_IQ2_XXS_R4: ++ case GGML_TYPE_IQ2_XS_R4: ++ case GGML_TYPE_IQ2_S_R4: ++ case GGML_TYPE_IQ3_XXS_R4: ++ case GGML_TYPE_IQ3_S_R4: ++ case GGML_TYPE_IQ1_S_R4: ++ case GGML_TYPE_IQ1_M_R4: ++ case GGML_TYPE_IQ2_BN_R4: return 4; ++ case GGML_TYPE_IQ4_XS_R8: ++ case GGML_TYPE_Q4_0_R8: ++ case GGML_TYPE_Q8_0_R8: ++ case GGML_TYPE_Q8_KV: ++ case GGML_TYPE_Q8_KV_R8: ++ case GGML_TYPE_Q8_1: ++ case GGML_TYPE_Q8_K_R8: return 8; ++ case GGML_TYPE_Q8_K_R16: ++ case GGML_TYPE_BF16_R16: return 16; ++ default: return 1; ++ } ++#endif ++} ++ ++ ++} ++ ++void iqk_quantize_any(int from_type, int to_type, ++ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, ++ uint64_t nb0, uint64_t nb1, uint64_t nb2, uint64_t nb3, ++ const void * x, void * y, void * work_buffer, ++ to_float_t to_float, from_float_t from_float, int ith, int nth) { ++ auto type_x = ggml_type(from_type); ++ GGML_ASSERT(ggml_type_size(type_x) == nb0); ++ auto type_y = ggml_type(to_type); ++ auto row_size_y = ggml_row_size(type_y, ne0); ++ auto n_interleaved = num_rows(type_y); ++ GGML_ASSERT(ne1 % n_interleaved == 0); ++ int64_t ne1i = ne1/n_interleaved; ++ int64_t nrows = ne1i*ne2*ne3; ++ int64_t nrows_per_thread = (nrows + nth - 1)/nth; ++ int64_t first_row = nrows_per_thread*ith; ++ if (first_row >= nrows) return; ++ int64_t last_row = std::min(first_row + nrows_per_thread, nrows); ++ for (int64_t row = first_row; row < last_row; ++row) { ++ int64_t i3 = row/(ne1i*ne2); ++ int64_t i2 = (row - i3*ne1i*ne2)/ne1i; ++ int64_t i1 = row - i3*ne1i*ne2 - i2*ne1i; ++ auto cx = (const char *)x + i1*n_interleaved*nb1 + i2*nb2 + i3*nb3; ++ auto cy = (char *)y + (i3*ne1*ne2 + i2*ne1 + i1*n_interleaved)*row_size_y; ++ // TODO: special case common types such as f16, q8_0 ++ // (although the performance gains may be too small to justify the added complexity) ++ if (type_x != GGML_TYPE_F32) { ++ to_float((const void *)cx, (float *)work_buffer, ne0*n_interleaved); ++ from_float((const float *)work_buffer, (void *)cy, ne0*n_interleaved); ++ } else { ++ from_float((const float *)cx, (void *)cy, ne0*n_interleaved); ++ } ++ } ++} ++ ++ ++size_t quantize_iq1_bn(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data *) { ++ IQ1BNQuantizer iq1bn; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ1_BN, n_per_row); ++ auto qrow = (char *)dst; ++ for (int row = 0; row < nrows; ++row) { ++ iq1bn.quantize_one_row_1bn(src + row*n_per_row, (block_iq1_bn *)qrow, n_per_row, imatrix); ++ qrow += row_size; ++ } ++ return nrows*row_size; ++} ++ ++void quantize_row_iq1_bn_ref(const float * x, block_iq1_bn * y, int64_t k) { ++ quantize_iq1_bn(x, y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq1_bn(const float * x, void * y, int64_t k) { ++ quantize_iq1_bn(x, y, 1, k, nullptr, nullptr); ++} ++ ++void dequantize_row_iq1_bn(const block_iq1_bn * x, float * y, int64_t k) { ++ assert(k%QK_IQ1BN == 0); ++ int nblock = k / QK_IQ1BN; ++ ++ for (int i = 0; i < nblock; ++i) { ++ uint8_t extra = x[i].extra; ++ auto ql = x[i].ql; ++ for (int i16 = 0; i16 < QK_IQ1BN/16; ++i16) { ++ for (int k = 0; k < 3; ++k) { ++ for (int j = 0; j < 5; ++j) { ++ uint8_t v = ql[k]*IQ1BNQuantizer::k_mult[j]; ++ int8_t vs = ((v + (v >> 1)) >> 7); ++ *y++ = vs - 1; ++ } ++ } ++ ql += 3; ++ uint8_t v = extra*IQ1BNQuantizer::k_mult[i16]; ++ int8_t vs = ((v + (v >> 1)) >> 7); ++ *y++ = vs - 1; ++ } ++ } ++} ++ ++size_t quantize_iq2_bn(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ IQ1BNQuantizer iq1bn; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_BN, n_per_row); ++ auto qrow = (char *)dst; ++ for (int row = 0; row < nrows; ++row) { ++ iq1bn.quantize_one_row_2bn(src + row*n_per_row, (block_iq2_bn *)qrow, n_per_row, imatrix); ++ qrow += row_size; ++ } ++ return nrows*row_size; ++} ++ ++void quantize_row_iq2_bn_ref(const float * x, block_iq2_bn * y, int64_t k) { ++ quantize_iq2_bn(x, y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_bn(const float * x, void * y, int64_t k) { ++ quantize_iq2_bn(x, y, 1, k, nullptr, nullptr); ++} ++ ++void dequantize_row_iq2_bn(const block_iq2_bn * x, float * y, int64_t k) { ++ assert(k%QK_IQ1BN == 0); ++ int nblock = k / QK_IQ1BN; ++ ++ auto d1 = 1.f, d2 = 0.25f, d3 = d2*0.25f, d4 = d3*0.25f; ++ auto m = -1.f; ++ constexpr int Nj = QK_IQ1BN/4; ++ for (int i = 0; i < nblock; ++i) { ++ for (int j = 0; j < Nj; ++j) { ++ y[j+ 0] = d1*(x[i].qs[j] & 0x03) + m; ++ y[j+1*Nj] = d2*(x[i].qs[j] & 0x0c) + m; ++ y[j+2*Nj] = d3*(x[i].qs[j] & 0x30) + m; ++ y[j+3*Nj] = d4*(x[i].qs[j] & 0xc0) + m; ++ } ++ y += QK_IQ1BN; ++ } ++} ++ ++namespace { ++inline int8_t iq1bn_dequant(uint8_t q, int i) { ++ uint8_t v = IQ1BNQuantizer::k_mult[i]*q; ++ //int8_t vs = (v + (v << 1)) >> 8; ++ int8_t vs = 3*v >> 8; ++ return vs - 1; ++} ++} ++ ++static const int8_t iq1bn_values[1280] = { ++ -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 0, -1, -1, -1, 0, 0, -1, -1, -1, 1, 0, ++ -1, -1, -1, -1, 1, -1, -1, -1, 0, 1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, 0, -1, -1, 0, -1, 0, -1, -1, 1, -1, 0, -1, ++ -1, -1, 0, 0, -1, -1, 0, 0, 0, -1, -1, 1, 0, 0, -1, -1, -1, 1, 0, -1, -1, 0, 1, 0, -1, -1, 1, 1, 0, -1, -1, -1, ++ -1, 1, -1, -1, 0, 0, 0, 0, 0, 0, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1, 0, 1, -1, -1, 0, 0, 1, -1, -1, 1, 0, 1, ++ -1, -1, -1, 1, 1, -1, -1, 0, 1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 0, -1, 0, -1, -1, 0, -1, 1, -1, -1, 0, -1, ++ -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 1, 0, -1, 0, -1, -1, 1, -1, 0, -1, 0, 1, -1, 0, -1, 1, 1, -1, 0, -1, -1, -1, ++ 0, 0, -1, 0, -1, 0, 0, -1, 0, 0, 0, 0, 0, 1, -1, 0, 0, -1, -1, 0, 0, 0, -1, 0, 0, 0, 0, -1, 1, 0, 0, 0, ++ -1, -1, 1, 0, 0, -1, 0, 1, 0, 0, -1, 1, 1, 0, 0, -1, -1, -1, 1, 0, -1, 0, -1, 1, 0, -1, 1, -1, 1, 0, -1, -1, ++ 0, 1, 0, -1, 0, 0, 1, 0, -1, 1, 0, 1, 0, -1, -1, 1, 1, 0, -1, 0, 1, 1, 0, -1, 1, 1, 1, 0, -1, -1, -1, -1, ++ 1, -1, 0, -1, -1, 1, -1, 1, -1, -1, 1, -1, 0, 0, 0, 0, 0, -1, 0, -1, 1, -1, 0, 0, -1, 1, -1, 1, 0, -1, 1, -1, ++ -1, 1, -1, 1, -1, 0, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, 0, 1, -1, 0, -1, 0, 1, -1, 1, -1, 0, 1, -1, -1, 0, ++ 0, 1, -1, 0, 0, 0, 1, -1, 1, 0, 0, 1, -1, -1, 1, 0, 1, -1, 0, 1, 0, 1, -1, 1, 1, 0, 1, -1, -1, -1, 1, 1, ++ -1, 0, -1, 1, 1, -1, 1, -1, 1, 1, -1, 0, 0, 0, 0, 0, -1, 0, 1, 1, -1, 0, 0, 1, 1, -1, 1, 0, 1, 1, -1, -1, ++ 1, 1, 1, -1, 0, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, -1, -1, -1, 0, 0, -1, -1, -1, 0, 1, -1, -1, -1, 0, -1, 0, -1, ++ -1, 0, 0, 0, -1, -1, 0, 1, 0, -1, -1, 0, -1, 1, -1, -1, 0, 0, 1, -1, -1, 0, 1, 1, -1, -1, 0, -1, -1, 0, -1, 0, ++ 0, -1, 0, -1, 0, 1, -1, 0, -1, 0, -1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 1, 0, 0, -1, 0, -1, 1, ++ 0, -1, 0, 0, 1, 0, -1, 0, 1, 1, 0, -1, 0, -1, -1, 1, -1, 0, 0, -1, 1, -1, 0, 1, -1, 1, -1, 0, -1, 0, 1, -1, ++ 0, 0, 0, 1, -1, 0, 1, 0, 1, -1, 0, -1, 1, 1, -1, 0, 0, 1, 1, -1, 0, 1, 1, 1, -1, 0, -1, -1, -1, 0, 0, 0, ++ -1, -1, 0, 0, 1, -1, -1, 0, 0, -1, 0, -1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 1, 0, -1, 0, 0, -1, 1, -1, ++ 0, 0, 0, 1, -1, 0, 0, 1, 1, -1, 0, 0, -1, -1, 0, 0, 0, 0, -1, 0, 0, 0, 1, -1, 0, 0, 0, -1, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, -1, -1, 1, 0, 0, 0, -1, ++ 1, 0, 0, 1, -1, 1, 0, 0, -1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, -1, 1, 1, 0, ++ 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, -1, -1, -1, 1, 0, 0, -1, -1, 1, 0, 1, -1, -1, 1, 0, -1, 0, -1, 1, 0, 0, ++ 0, -1, 1, 0, 1, 0, -1, 1, 0, -1, 1, -1, 1, 0, 0, 1, -1, 1, 0, 1, 1, -1, 1, 0, -1, -1, 0, 1, 0, 0, -1, 0, ++ 1, 0, 1, -1, 0, 1, 0, -1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, -1, 1, 0, 1, 0, ++ 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, -1, -1, 1, 1, 0, 0, -1, 1, 1, 0, 1, -1, 1, 1, 0, -1, 0, 1, 1, 0, 0, 0, ++ 1, 1, 0, 1, 0, 1, 1, 0, -1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, -1, -1, -1, -1, 1, 0, -1, -1, -1, ++ 1, 1, -1, -1, -1, 1, -1, 0, -1, -1, 1, 0, 0, -1, -1, 1, 1, 0, -1, -1, 1, -1, 1, -1, -1, 1, 0, 0, 0, 0, 0, 0, ++ 1, -1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 0, -1, 1, 0, -1, 0, -1, 1, 1, -1, 0, -1, 1, -1, 0, 0, -1, 1, 0, 0, 0, ++ -1, 1, 1, 0, 0, -1, 1, -1, 1, 0, -1, 1, 0, 1, 0, -1, 1, 1, 1, 0, -1, 1, -1, -1, 1, -1, 1, 0, -1, 1, -1, 1, ++ 1, -1, 1, -1, 1, -1, 0, 1, -1, 1, 0, 0, 1, -1, 1, 1, 0, 1, -1, 1, -1, 1, 1, -1, 1, 0, 0, 0, 0, 0, 0, 1, ++ 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, -1, 0, 1, 0, -1, -1, 0, 1, 1, -1, -1, 0, 1, -1, 0, -1, 0, 1, 0, 0, -1, 0, ++ 1, 1, 0, -1, 0, 1, -1, 1, -1, 0, 1, 0, 1, -1, 0, 1, 1, 1, -1, 0, 1, -1, -1, 0, 0, 1, 0, -1, 0, 0, 1, 1, ++ -1, 0, 0, 1, -1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, -1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, ++ 0, 0, 1, 1, 0, 0, 1, -1, -1, 1, 0, 1, 0, -1, 1, 0, 1, 1, -1, 1, 0, 1, -1, 0, 1, 0, 1, 0, 0, 1, 0, 1, ++ 1, 0, 1, 0, 1, -1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, -1, -1, -1, 1, 1, 0, -1, -1, 1, 1, 1, -1, ++ -1, 1, 1, -1, 0, -1, 1, 1, 0, 0, -1, 1, 1, 1, 0, -1, 1, 1, -1, 1, -1, 1, 1, 0, 1, -1, 1, 1, 1, 1, -1, 1, ++ 1, 0, 0, 0, 0, 0, -1, -1, 0, 1, 1, 0, -1, 0, 1, 1, 1, -1, 0, 1, 1, -1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, ++ 0, 0, 1, 1, -1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, -1, -1, 1, 1, 1, 0, -1, 1, 1, 1, 1, -1, 1, ++ 1, 1, -1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, -1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ++}; ++ ++void ggml_vec_dot_iq1_bn_q8_K64(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(nrc); ++ ++ static_assert(QK_IQ1BN == 64, "This dot product implementation for iq1_bn requires a block size of 64"); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ1_BN, vx, 0, GGML_TYPE_Q8_K64, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ const block_iq1_bn * x = (const block_iq1_bn *)vx; ++ ++ const float * d8 = (const float *)vy; ++ const int8_t * q8 = (const int8_t *)(d8 + 4); ++ int nblock = n / QK_IQ1BN; ++ ++ int sumi[8] = {}; ++ int8_t q1[16]; ++ ++ for (int ii = 0; ii < nblock; ii += 32) { ++ int16_t sum16[8] = {}; ++ int nb = std::min(ii + 32, nblock); ++ for (int i = ii; i < nb; ++i) { ++ auto ql = x[i].ql; ++ const int8_t * extra = iq1bn_values + 5*x[i].extra; ++ for (int i16 = 0; i16 < QK_IQ1BN/16; ++i16) { ++ for (int k = 0; k < 3; ++k) { ++ uint8_t q = *ql++; ++ const int8_t * vs = iq1bn_values + 5*q; ++ for (int j = 0; j < 5; ++j) q1[5*k+j] = vs[j]; ++ } ++ q1[15] = extra[i16]; ++ // We collect 8 q8 values per block into each element of sum16 ++ // => 32 x 8 = 256 values in each loop over i, so this cannot overflow the int16_t range ++ // (q8 is in -127...127, and hence the sum is in -32512...32512 ++ for (int j = 0; j < 8; ++j) sum16[j] += q8[2*j+0]*q1[2*j+0] + q8[2*j+1]*q1[2*j+1]; ++ q8 += 16; ++ } ++ } ++ for (int j = 0; j < 8; ++j) sumi[j] += sum16[j]; ++ } ++ ++ *s = d8[0] * (sumi[0] + sumi[1]) + d8[1] * (sumi[2] + sumi[3]) + d8[2] * (sumi[4] + sumi[5]) + d8[3] * (sumi[6] + sumi[7]); ++} ++ ++void vec_dot_iq2_bn_q8_K64(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(nrc); ++ ++ static_assert(QK_IQ1BN == 64, "This dot product implementation for iq2_bn requires a block size of 64"); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_BN, vx, 0, GGML_TYPE_Q8_K64, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ constexpr int Nj = QK_IQ1BN/4; ++ ++ const block_iq2_bn * x = (const block_iq2_bn *)vx; ++ int nblock = n / QK_IQ1BN; ++ ++ const float * d = (const float *)vy; ++ const int8_t * q8 = (const int8_t *)(d + 4); ++ ++ int sum[16] = { }; ++ int sum0[4] = { }; ++ ++ for (int i = 0; i < nblock; ++i) { ++ for (int j = 0; j < Nj/4; ++j) { ++ for (int l = 0; l < 4; ++l) { ++ sum[4*j + 0] += q8[4*j + l + 0] * (x[i].qs[4*j+l] & 0x03); ++ sum[4*j + 1] += q8[4*j + l + 1*Nj] * (x[i].qs[4*j+l] & 0x0c); ++ sum[4*j + 2] += q8[4*j + l + 2*Nj] * (x[i].qs[4*j+l] & 0x30); ++ sum[4*j + 3] += q8[4*j + l + 3*Nj] * (x[i].qs[4*j+l] & 0xc0); ++ sum0[j] += q8[4*j + l] + q8[4*j + l + 1*Nj] + q8[4*j + l + 2*Nj] + q8[4*j + l + 3*Nj]; ++ } ++ } ++ q8 += QK_IQ1BN; ++ } ++ ++ float sumf = 0; ++ for (int j = 0; j < 4; ++j) { ++ sumf += d[j] * (sum[4*j + 0] + 0.25f*sum[4*j + 1] + 0.0625*sum[4*j + 2] + 0.015625*sum[4*j + 3] - sum0[j]); ++ } ++ *s = sumf; ++ ++} ++ ++void quantize_row_q8_K64_ref(const float * x, block_q8_K64 * y, int64_t k) { ++ ++ GGML_ASSERT(k >= 8*QK_IQ1BN); ++ ++ float * dptr = (float *)y; ++ auto qs = (int8_t *)(dptr + 8); ++#ifdef __ARM_NEON ++ static const uint8_t k_shuffle[16] = {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60}; ++ auto shuffle = vld1q_u8(k_shuffle); ++ float32x4_t max[4] = { }; ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ auto val = vld1q_f32(x + j + 4*i); ++ val = vabsq_f32(val); ++ max[i] = vmaxq_f32(max[i], val); ++ } ++ } ++ float32x4_t vid[4]; ++ for (int i = 0; i < 4; ++i) { ++ dptr[i] = vmaxvq_f32(max[i])/127; ++ float id = dptr[i] > 0 ? 1/dptr[i] : 0.f; ++ vid[i] = vdupq_n_f32(id); ++ } ++ int8x16x4_t q; ++ int32x4_t qsum = {}; ++ const int8x16_t m1 = vdupq_n_s8(1); ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ auto val = vld1q_f32(x + j + 4*i); ++ val = vmulq_f32(vid[i], val); ++ auto ival = vcvtnq_s32_f32(val); ++ q.val[i] = vreinterpretq_s8_s32(ival); ++ } ++ auto qi = vqtbl4q_s8(q, shuffle); ++ qsum = ggml_vdotq_s32(qsum, qi, m1); ++ vst1q_s8(qs, qi); ++ qs += 16; ++ } ++ auto sumf = vmulq_f32(vld1q_f32(dptr), vcvtq_f32_s32(qsum)); ++ vst1q_f32(dptr + 4, sumf); ++#elif defined __AVX__ ++ __m128 max[4] = {}; ++ __m128 sign_bit = _mm_set1_ps(-0.f); ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ auto val = _mm_loadu_ps(x + j + 4*i); ++ val = _mm_andnot_ps(sign_bit, val); ++ max[i] = _mm_max_ps(max[i], val); ++ } ++ } ++ __m128 vid[4]; ++ for (int i = 0; i < 4; ++i) { ++ max[i] = _mm_max_ps(max[i], _mm_movehl_ps(max[i], max[i])); ++ max[i] = _mm_max_ss(max[i], _mm_movehdup_ps(max[i])); ++ float maxi = _mm_cvtss_f32(max[i]); ++ dptr[i] = maxi/127; ++ float id = dptr[i] > 0 ? 1/dptr[i] : 0.f; ++ vid[i] = _mm_set1_ps(id); ++ } ++ __m128i q[4]; ++ __m128i sums = _mm_setzero_si128(); ++ __m128i m1_8 = _mm_set1_epi8(1); ++ __m128i m1_16 = _mm_set1_epi16(1); ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ auto val = _mm_loadu_ps(x + j + 4*i); ++ val = _mm_round_ps(_mm_mul_ps(vid[i], val), _MM_ROUND_NEAREST); ++ q[i] = _mm_cvtps_epi32(val); ++ } ++ auto q1 = _mm_packs_epi32(q[0], q[1]); ++ auto q2 = _mm_packs_epi32(q[2], q[3]); ++ auto qi = _mm_packs_epi16(q1, q2); ++ auto aux = _mm_maddubs_epi16(m1_8, qi); ++ sums = _mm_add_epi32(sums, _mm_madd_epi16(m1_16, aux)); ++ _mm_storeu_si128((__m128i *)qs, qi); ++ qs += 16; ++ } ++ auto minus = _mm_mul_ps(_mm_loadu_ps(dptr), _mm_cvtepi32_ps(sums)); ++ _mm_storeu_ps(dptr + 4, minus); ++#else ++ float aux[4] = {0.f, 0.f, 0.f, 0.f}; ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ for (int l = 0; l < 4; ++l) { ++ float ax = fabsf(x[j+4*i+l]); ++ aux[i] = std::max(aux[i], ax); ++ } ++ } ++ } ++ for (int i = 0; i < 4; ++i) { ++ dptr[i] = aux[i]/127; ++ aux[i] = dptr[i] > 0 ? 1/dptr[i] : 0.f; ++ } ++ int32_t sum[4] = {}; ++ for (int j = 0; j < k; j += 16) { ++ for (int i = 0; i < 4; ++i) { ++ for (int l = 0; l < 4; ++l) { ++ qs[j+4*i+l] = nearest_int(aux[i]*x[j+4*i+l]); ++ sum[i] += qs[j+4*i+l]; ++ } ++ } ++ } ++ for (int i = 0; i < 4; ++i) dptr[4+i] = dptr[i]*sum[i]; ++#endif ++} ++ ++void quantize_row_q8_K64(const float * x, void * y, int64_t k) { ++ quantize_row_q8_K64_ref(x, (block_q8_K64 *)y, k); ++} ++ ++void quantize_row_q8_K16(const float * x, void * vy, int64_t nk) { ++ float * dptr = (float *)vy; ++ int8_t * qy = (int8_t *)(dptr + 5); ++ int n64 = nk / 64; ++#ifdef z__AVX2__ ++ __m256 sign_bit = _mm256_set1_ps(-0.f); ++ __m256 vmax[4] = {}; ++ __m256 vsum[4] = {}; ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ auto v1 = _mm256_loadu_ps(x + 64*i64 + 16*k + 0); ++ auto v2 = _mm256_loadu_ps(x + 64*i64 + 16*k + 8); ++ vsum[k] = _mm256_add_ps(vsum[k], _mm256_add_ps(v1, v2)); ++ v1 = _mm256_andnot_ps(sign_bit, v1); ++ v2 = _mm256_andnot_ps(sign_bit, v2); ++ vmax[k] = _mm256_max_ps(vmax[k], _mm256_max_ps(v1, v2)); ++ } ++ } ++ __m256 sum = _mm256_add_ps(_mm256_add_ps(vsum[0], vsum[1]), _mm256_add_ps(vsum[2], vsum[3])); ++ dptr[4] = hsum_float_8(sum); ++ for (int k = 0; k < 4; ++k) { ++ float max = hmax_f32_8(vmax[k]); ++ dptr[k] = max/127; ++ vmax[k] = _mm256_set1_ps(dptr[k] > 0 ? 1/dptr[k] : 0.f); ++ } ++ __m256i ival[8]; ++ const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ __m256 v0 = _mm256_mul_ps(vmax[k], _mm256_loadu_ps(x + 64*i64 + 16*k + 0)); ++ __m256 v1 = _mm256_mul_ps(vmax[k], _mm256_loadu_ps(x + 64*i64 + 16*k + 8)); ++ v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST); ++ v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST); ++ ival[2*k+0] = _mm256_cvtps_epi32(v0); ++ ival[2*k+1] = _mm256_cvtps_epi32(v1); ++ } ++ for (int k = 0; k < 2; ++k) { ++ auto i0 = _mm256_packs_epi32(ival[4*k+0], ival[4*k+1]); ++ auto i1 = _mm256_packs_epi32(ival[4*k+2], ival[4*k+3]); ++ i0 = _mm256_packs_epi16(i0, i1); ++ i0 = _mm256_permutevar8x32_epi32(i0, perm); ++ _mm256_storeu_si256((__m256i *)qy, i0); ++ qy += 32; ++ } ++ } ++#elif defined z__ARM_NEON ++ static const uint8_t k_shuffle[16] = {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60}; ++ auto shuffle = vld1q_u8(k_shuffle); ++ float32x4_t vmax[4] = {}; ++ float32x4_t vsum[4] = {}; ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ auto v = vld1q_f32_x4(x + 64*i64 + 16*k); ++ vsum[k] = vaddq_f32(vsum[k], vaddq_f32(v.val[0], v.val[1])); ++ vsum[k] = vaddq_f32(vsum[k], vaddq_f32(v.val[2], v.val[3])); ++ vmax[k] = vmaxq_f32(vmax[k], vmaxq_f32(vabsq_f32(v.val[0]), vabsq_f32(v.val[1]))); ++ vmax[k] = vmaxq_f32(vmax[k], vmaxq_f32(vabsq_f32(v.val[2]), vabsq_f32(v.val[3]))); ++ } ++ } ++ dptr[4] = vaddvq_f32(vaddq_f32(vaddq_f32(vsum[0], vsum[1]), vaddq_f32(vsum[2], vsum[3]))); ++ for (int k = 0; k < 4; ++k) { ++ float max = vmaxvq_f32(vmax[k]); ++ dptr[k] = max/127; ++ vmax[k] = vdupq_n_f32(dptr[k] > 0 ? 1/dptr[k] : 0.f); ++ } ++ int8x16x4_t q; ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ auto v = vld1q_f32_x4(x + 64*i64 + 16*k); ++ for (int j = 0; j < 4; ++j) { ++ q.val[j] = vreinterpretq_s8_s32(vcvtnq_s32_f32(vmulq_f32(vmax[k], v.val[j]))); ++ } ++ auto qi = vqtbl4q_s8(q, shuffle); ++ vst1q_s8(qy, qi); ++ qy += 16; ++ } ++ } ++#else ++ float amax[4] = {0.f, 0.f, 0.f, 0.f}; ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ for (int j = 0; j < 16; ++j) { ++ float ax = std::abs(x[64*i64 + 16*k + j]); ++ amax[k] = std::max(amax[k], ax); ++ } ++ } ++ } ++ for (int k = 0; k < 4; ++k) { ++ dptr[k] = amax[k]/127; ++ amax[k] = dptr[k] > 0 ? 1/dptr[k] : 0.f; ++ } ++ int sumi[4] = {}; ++ for (int i64 = 0; i64 < n64; ++i64) { ++ for (int k = 0; k < 4; ++k) { ++ for (int j = 0; j < 16; ++j) { ++ int ix = nearest_int(amax[k]*x[64*i64 + 16*k + j]); ++ sumi[k] += ix; ++ qy[64*i64 + 16*k + j] = ix; ++ } ++ } ++ } ++ dptr[4] = dptr[0]*sumi[0] + dptr[1]*sumi[1] + dptr[2]*sumi[2] + dptr[3]*sumi[3]; ++#endif ++} ++ ++void iqk_quantize_q4_0(const float * x, void * vy, int64_t k) { ++ const int nb = k / QK4_0; ++ auto y = (block_q4_0 *)vy; ++#ifdef __AVX2__ ++ static_assert(QK4_0 == 32); ++ __m256 vx[4], rx[4]; ++ __m256i ix[4]; ++ auto v7 = _mm256_set1_ps(7.0f); ++ auto perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ vx[k] = _mm256_loadu_ps(x + 8*k); ++ } ++ auto max1 = _mm256_max_ps(vx[0], vx[1]); ++ auto max2 = _mm256_max_ps(vx[2], vx[3]); ++ auto vmax = _mm256_max_ps(max1, max2); ++ auto min1 = _mm256_min_ps(vx[0], vx[1]); ++ auto min2 = _mm256_min_ps(vx[2], vx[3]); ++ auto vmin = _mm256_min_ps(min1, min2); ++ float max = hmax_float_8(vmax); ++ float min = hmin_float_8(vmin); ++ float amax = std::abs(max); ++ float amin = std::abs(min); ++ float d, id; ++ if (amax > amin) { ++ d = max / -8; ++ id = amax > 1e-13f ? 1/d : 0.0f; ++ } else { ++ d = min / -8; ++ id = amin > 1e-13f ? 1/d : 0.0f; ++ } ++ auto vid = _mm256_set1_ps(id); ++ auto vsumqx = _mm256_setzero_ps(); ++ auto vsumq2 = _mm256_setzero_ps(); ++ for (int k = 0; k < 4; ++k) { ++ rx[k] = _mm256_mul_ps(vid, vx[k]); ++ rx[k] = _mm256_round_ps(rx[k], _MM_ROUND_NEAREST); ++ rx[k] = _mm256_min_ps(rx[k], v7); ++ ix[k] = _mm256_cvtps_epi32(rx[k]); ++ ix[k] = _mm256_add_epi32(ix[k], _mm256_set1_epi32(8)); ++ auto w = _mm256_mul_ps(vx[k], vx[k]); ++ auto wr = _mm256_mul_ps(w, rx[k]); ++ vsumqx = _mm256_fmadd_ps(wr, vx[k], vsumqx); ++ vsumq2 = _mm256_fmadd_ps(wr, rx[k], vsumq2); ++ } ++ auto sumq2 = hsum_float_8(vsumq2); ++ if (sumq2 > 0) { ++ auto sumqx = hsum_float_8(vsumqx); ++ d = sumqx/sumq2; ++ } ++ y[ib].d = GGML_FP32_TO_FP16(d); ++ auto i0 = _mm256_packs_epi32(ix[0], ix[1]); ++ auto i2 = _mm256_packs_epi32(ix[2], ix[3]); ++ i0 = _mm256_packs_epi16(i0, i2); ++ i0 = _mm256_permutevar8x32_epi32(i0, perm); ++ auto q = _mm_or_si128(_mm256_castsi256_si128(i0), _mm_slli_epi16(_mm256_extracti128_si256(i0, 1), 4)); ++ _mm_storeu_si128((__m128i *)y[ib].qs, q); ++ x += QK4_0; ++ } ++#else ++ for (int ib = 0; ib < nb; ++ib) { ++ float max = 0, amax = 0; ++ for (int j = 0; j < QK4_0; ++j) { ++ float ax = std::abs(x[j]); ++ if (ax > amax) { ++ amax = ax; max = x[j]; ++ } ++ } ++ float d = max / -8; ++ float id = amax > 1e-13f ? 1/d : 0.0f; ++ float sumqx = 0, sumq2 = 0; ++ for (int j = 0; j < QK4_0/2; ++j) { ++ float v0 = x[j], v1 = x[j+QK4_0/2]; ++ int i0 = nearest_int(id*v0), i1 = nearest_int(id*v1); ++ i0 = std::min(i0, 7); ++ i1 = std::min(i1, 7); ++ float w0 = v0*v0, w1 = v1*v1; ++ sumqx += w0*i0*v0 + w1*i1*v1; ++ sumq2 += w0*i0*i0 + w1*i1*i1; ++ y[ib].qs[j] = (i0 + 8) | ((i1 + 8) << 4); ++ } ++ if (sumq2 > 0) d = sumqx/sumq2; ++ y[ib].d = GGML_FP32_TO_FP16(d); ++ x += QK4_0; ++ } ++#endif ++} ++ ++void quantize_row_q8_0_x4(const float * x, void * vy, int64_t k) { ++ const int nb = k / QK8_0; ++ const int nb4 = 4*(nb/4); ++ ++ block_q8_0 * y = (block_q8_0 *)vy; ++ block_q8_0_x4 * y4 = (block_q8_0_x4 *)vy; ++#if defined(__aarch64__) ++ for (int i = 0; i < nb; i++) { ++ int i4 = i/4, ir = i%4; ++ float32x4_t srcv [8]; ++ float32x4_t asrcv[8]; ++ float32x4_t amaxv[8]; ++ ++ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j); ++ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]); ++ ++ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]); ++ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]); ++ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]); ++ ++ const float amax = vmaxvq_f32(amaxv[0]); ++ ++ const float d = amax / ((1 << 7) - 1); ++ const float id = d ? 1.0f/d : 0.0f; ++ ++ if (i < nb4) { ++ y4[i4].d[ir] = GGML_FP32_TO_FP16(d); ++ } else { ++ y[i].d = GGML_FP32_TO_FP16(d); ++ } ++ ++ for (int j = 0; j < 8; j++) { ++ const float32x4_t v = vmulq_n_f32(srcv[j], id); ++ const int32x4_t vi = vcvtnq_s32_f32(v); ++ ++ if (i < nb4) { ++ y4[i4].qs[32*ir + 4*j + 0] = vgetq_lane_s32(vi, 0); ++ y4[i4].qs[32*ir + 4*j + 1] = vgetq_lane_s32(vi, 1); ++ y4[i4].qs[32*ir + 4*j + 2] = vgetq_lane_s32(vi, 2); ++ y4[i4].qs[32*ir + 4*j + 3] = vgetq_lane_s32(vi, 3); ++ } else { ++ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0); ++ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1); ++ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2); ++ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3); ++ } ++ } ++ } ++#else ++ for (int i = 0; i < nb; i++) { ++ int i4 = i/4, ir = i%4; ++ // Load elements into 4 AVX vectors ++ __m256 v0 = _mm256_loadu_ps( x ); ++ __m256 v1 = _mm256_loadu_ps( x + 8 ); ++ __m256 v2 = _mm256_loadu_ps( x + 16 ); ++ __m256 v3 = _mm256_loadu_ps( x + 24 ); ++ x += 32; ++ ++ const __m256 signBit = _mm256_set1_ps( -0.0f ); ++ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) ); ++ ++ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); ++ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); ++ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); ++ const float maxScalar = _mm_cvtss_f32( max4 ); ++ ++ const float d = maxScalar / 127.f; ++ if (i < nb4) { ++ y4[i4].d[ir] = GGML_FP32_TO_FP16(d); ++ } else { ++ y[i].d = GGML_FP32_TO_FP16(d); ++ } ++ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; ++ const __m256 mul = _mm256_set1_ps( id ); ++ ++ v0 = _mm256_mul_ps( v0, mul ); ++ v1 = _mm256_mul_ps( v1, mul ); ++ v2 = _mm256_mul_ps( v2, mul ); ++ v3 = _mm256_mul_ps( v3, mul ); ++ ++ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); ++ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); ++ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); ++ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); ++ ++ __m256i i0 = _mm256_cvtps_epi32( v0 ); ++ __m256i i1 = _mm256_cvtps_epi32( v1 ); ++ __m256i i2 = _mm256_cvtps_epi32( v2 ); ++ __m256i i3 = _mm256_cvtps_epi32( v3 ); ++ ++ // Convert int32 to int16 ++ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 ++ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31 ++ // Convert int16 to int8 ++ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 ++ ++ // We got our precious signed bytes, but the order is now wrong ++ // These AVX2 pack instructions process 16-byte pieces independently ++ // The following instruction is fixing the order ++ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); ++ i0 = _mm256_permutevar8x32_epi32( i0, perm ); ++ ++ if (i < nb4) { ++ _mm256_storeu_si256((__m256i *)y4[i4].qs + ir, i0); ++ } else { ++ _mm256_storeu_si256((__m256i *)y[i].qs, i0); ++ } ++ } ++#endif ++} ++ ++namespace { ++template ++void quantize_row_q8_1_x4_T(const float * x, Block * y, int64_t k) { ++ assert(k % QK8_1 == 0); ++ const int nb = k / QK8_1; ++ ++ const int nb4 = 4*(nb/4); ++ Block_x4 * y4 = (Block_x4 *)y; ++#if defined(__aarch64__) ++ for (int i = 0; i < nb; i++) { ++ int i4 = i/4, ir = i%4; ++ float32x4_t srcv [8]; ++ float32x4_t asrcv[8]; ++ float32x4_t amaxv[8]; ++ ++ for (int j = 0; j < 8; j++) srcv[j] = vld1q_f32(x + i*32 + 4*j); ++ for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]); ++ ++ for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]); ++ for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]); ++ for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]); ++ ++ const float amax = vmaxvq_f32(amaxv[0]); ++ ++ const float d = amax / ((1 << 7) - 1); ++ const float id = d ? 1.0f/d : 0.0f; ++ ++ if (i < nb4) { ++ y4[i4].d[ir] = GGML_FP32_TO_FP16(d); ++ } else { ++ y[i].d = GGML_FP32_TO_FP16(d); ++ } ++ ++ int32x4_t accv = vdupq_n_s32(0); ++ ++ for (int j = 0; j < 8; j++) { ++ const float32x4_t v = vmulq_n_f32(srcv[j], id); ++ const int32x4_t vi = vcvtnq_s32_f32(v); ++ ++ if (i < nb4) { ++ y4[i4].qs[QK8_1*ir + 4*j + 0] = vgetq_lane_s32(vi, 0); ++ y4[i4].qs[QK8_1*ir + 4*j + 1] = vgetq_lane_s32(vi, 1); ++ y4[i4].qs[QK8_1*ir + 4*j + 2] = vgetq_lane_s32(vi, 2); ++ y4[i4].qs[QK8_1*ir + 4*j + 3] = vgetq_lane_s32(vi, 3); ++ } else { ++ y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0); ++ y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1); ++ y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2); ++ y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3); ++ } ++ ++ accv = vaddq_s32(accv, vi); ++ } ++ ++ if constexpr (std::is_same_v) { ++ if (i < nb4) { ++ y4[i4].d[ir+4] = GGML_FP32_TO_FP16(d * vaddvq_s32(accv)); ++ } else { ++ y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv)); ++ } ++ } else { ++ if (i < nb4) { ++ y4[i4].d[ir+4] = GGML_FP32_TO_BF16(d * vaddvq_s32(accv)).bits; ++ } else { ++ y[i].s = GGML_FP32_TO_BF16(d * vaddvq_s32(accv)).bits; ++ } ++ } ++ } ++#else ++ for (int i = 0; i < nb; i++) { ++ int i4 = i/4, ir = i%4; ++ // Load elements into 4 AVX vectors ++ __m256 v0 = _mm256_loadu_ps( x ); ++ __m256 v1 = _mm256_loadu_ps( x + 8 ); ++ __m256 v2 = _mm256_loadu_ps( x + 16 ); ++ __m256 v3 = _mm256_loadu_ps( x + 24 ); ++ x += 32; ++ ++ // Compute max(abs(e)) for the block ++ const __m256 signBit = _mm256_set1_ps( -0.0f ); ++ __m256 maxAbs = _mm256_andnot_ps( signBit, v0 ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) ); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) ); ++ ++ __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); ++ max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); ++ max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); ++ const float max_scalar = _mm_cvtss_f32( max4 ); ++ ++ // Quantize these floats ++ float d = max_scalar / 127.f; ++ if constexpr (std::is_same_v) { ++ if (i < nb4) { ++ y4[i4].d[ir] = GGML_FP32_TO_FP16(d); ++ } else { ++ y[i].d = GGML_FP32_TO_FP16(d); ++ } ++ } else { ++ auto t = GGML_FP32_TO_BF16(d); ++ d = ggml_bf16_to_fp32(t); ++ if (i < nb4) { ++ y4[i4].d[ir] = t.bits; ++ } else { ++ y[i].d = t.bits; ++ } ++ } ++ const float id = d > 0 ? 1/d : 0.f; ++ const __m256 mul = _mm256_set1_ps( id ); ++ ++ // Apply the multiplier ++ v0 = _mm256_mul_ps( v0, mul ); ++ v1 = _mm256_mul_ps( v1, mul ); ++ v2 = _mm256_mul_ps( v2, mul ); ++ v3 = _mm256_mul_ps( v3, mul ); ++ ++ // Round to nearest integer ++ v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST ); ++ v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST ); ++ v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST ); ++ v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST ); ++ ++ // Convert floats to integers ++ __m256i i0 = _mm256_cvtps_epi32( v0 ); ++ __m256i i1 = _mm256_cvtps_epi32( v1 ); ++ __m256i i2 = _mm256_cvtps_epi32( v2 ); ++ __m256i i3 = _mm256_cvtps_epi32( v3 ); ++ ++ // Compute the sum of the quants and set y[i].s ++ int isum = hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))); ++ if constexpr (std::is_same_v) { ++ if (i < nb4) { ++ y4[i4].d[ir+4] = GGML_FP32_TO_FP16(d * isum); ++ } else { ++ y[i].s = GGML_FP32_TO_FP16(d * isum); ++ } ++ } else { ++ if (i < nb4) { ++ auto i16 = (int16_t *)y4[i4].d; ++ i16[ir+4] = isum; ++ } else { ++ auto i16 = (int16_t *)&y[i].s; ++ i16[0] = isum; ++ } ++ } ++ ++ // Convert int32 to int16 ++ i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 ++ i2 = _mm256_packs_epi32( i2, i3 ); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31 ++ // Convert int16 to int8 ++ i0 = _mm256_packs_epi16( i0, i2 ); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 ++ ++ // We got our precious signed bytes, but the order is now wrong ++ // These AVX2 pack instructions process 16-byte pieces independently ++ // The following instruction is fixing the order ++ const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); ++ i0 = _mm256_permutevar8x32_epi32( i0, perm ); ++ ++ if (i < nb4) { ++ _mm256_storeu_si256((__m256i *)y4[i4].qs + ir, i0); ++ } else { ++ _mm256_storeu_si256((__m256i *)y[i].qs, i0); ++ } ++ } ++#endif ++} ++} ++ ++void quantize_row_q8_1_x4(const float * x, void * vy, int64_t k) { ++ quantize_row_q8_1_x4_T(x, (block_q8_1 *)vy, k); ++} ++ ++void quantize_row_q8_2_x4(const float * x, void * vy, int64_t k) { ++ quantize_row_q8_1_x4_T(x, (block_q8_2 *)vy, k); ++} ++ ++// ++// ============================================== iq2_K ++// ++ ++namespace { ++ ++inline int best_index_iq2nl(const int8_t * values, float x) { ++ int idx = x < values[1] ? 0 : x > values[2] ? 2 : 1; ++ return x - values[idx] < values[idx+1] - x ? idx : idx + 1; ++} ++ ++void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ ++ constexpr int kBlockSize = 16; ++ ++ block_iq2_k * y = (block_iq2_k *)vy; ++ ++ float scales[QK_K/kBlockSize]; ++ float weight[kBlockSize]; ++ float sumx[kBlockSize+1], sumw[kBlockSize+1]; ++ float sw[QK_K/kBlockSize]; ++ int8_t Ls[QK_K/kBlockSize]; ++ ++ std::array, kBlockSize> pairs; ++ ++ const int8_t * shifted_values = iq2nl_values + 4; ++ ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq2_k)); ++ y[ibl].d = GGML_FP32_TO_FP16(0.f); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ ++ uint16_t extra = 0; ++ ++ float max_abs_scale = 0; ++ ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ sw[ib] = 0; ++ float amax = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ sw[ib] += weight[j]; ++ pairs[j] = {xb[j], j}; ++ float ax = std::abs(xb[j]); ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ std::sort(pairs.begin(), pairs.end()); ++ sumx[0] = sumw[0] = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ int jj = pairs[j].second; ++ sumw[j+1] = sumw[j] + weight[jj]; ++ sumx[j+1] = sumx[j] + weight[jj]*xb[jj]; ++ } ++ float best = 0, d = 0; ++ bool is_shifted = false; ++ float sumqx, sumq2; ++ for (int i1 = 0; i1 < kBlockSize; ++i1) { ++ for (int i2 = i1; i2 < kBlockSize; ++i2) { ++ for (int i3 = i2; i3 < kBlockSize; ++i3) { ++ sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[0] + (sumx[i2] - sumx[i1])*iq2nl_values[1] ++ + (sumx[i3] - sumx[i2])*iq2nl_values[2] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[3]; ++ sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[0]*iq2nl_values[0] + (sumw[i2] - sumw[i1])*iq2nl_values[1]*iq2nl_values[1] ++ + (sumw[i3] - sumw[i2])*iq2nl_values[2]*iq2nl_values[2] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[3]*iq2nl_values[3]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = false; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*shifted_values[0] + (sumx[i2] - sumx[i1])*shifted_values[1] ++ + (sumx[i3] - sumx[i2])*shifted_values[2] + (sumx[kBlockSize] - sumx[i3])*shifted_values[3]; ++ sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[0]*shifted_values[0] + (sumw[i2] - sumw[i1])*shifted_values[1]*shifted_values[1] ++ + (sumw[i3] - sumw[i2])*shifted_values[2]*shifted_values[2] + (sumw[kBlockSize] - sumw[i3])*shifted_values[3]*shifted_values[3]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = true; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[3] + (sumx[i2] - sumx[i1])*iq2nl_values[2] ++ + (sumx[i3] - sumx[i2])*iq2nl_values[1] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[0]; ++ sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[3]*iq2nl_values[3] + (sumw[i2] - sumw[i1])*iq2nl_values[2]*iq2nl_values[2] ++ + (sumw[i3] - sumw[i2])*iq2nl_values[1]*iq2nl_values[1] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[0]*iq2nl_values[0]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = false; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*shifted_values[3] + (sumx[i2] - sumx[i1])*shifted_values[2] ++ + (sumx[i3] - sumx[i2])*shifted_values[1] + (sumx[kBlockSize] - sumx[i3])*shifted_values[0]; ++ sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[3]*shifted_values[3] + (sumw[i2] - sumw[i1])*shifted_values[2]*shifted_values[2] ++ + (sumw[i3] - sumw[i2])*shifted_values[1]*shifted_values[1] + (sumw[kBlockSize] - sumw[i3])*shifted_values[0]*shifted_values[0]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = true; ++ } ++ } ++ } ++ } ++ scales[ib] = d; ++ if (is_shifted) extra |= (1 << ib); ++ ++ float abs_scale = fabsf(scales[ib]); ++ max_abs_scale = std::max(max_abs_scale, abs_scale); ++ } ++ ++ if (!max_abs_scale) continue; ++ float d = make_qx_quants(QK_K/kBlockSize, 8, scales, Ls, sw); ++ if (!d) continue; ++ ++ //float d = -max_scale/8; ++ y[ibl].extra = extra; ++ float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ int ls = nearest_int(id*scales[ib]); ++ ls = std::max(-8, std::min(7, ls)); ++ y[ibl].scales[ib/2] |= ((ls + 8) << 4*(ib%2)); ++ float dl = d * ls; ++ if (dl) { ++ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq2nl_values; ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float idl = 1/dl; ++ int ib32 = ib/2; ++ int offset = 16*(ib%2); ++ uint8_t * qs = y[ibl].qs + 32*(ib32/4) + offset; ++ for (int j = 0; j < 16; ++j) { ++ const float al = idl*xb[j]; ++ int ibest = best_index_iq2nl(block_values, al); ++ qs[j] |= (ibest << 2*(ib32%4)); ++ float w = weight[j]; ++ float q = block_values[ibest]*ls; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ y[ibl].d = GGML_FP32_TO_FP16(1.030f*(sumq2 > 0 ? sumqx/sumq2 : d)); ++ ++ } ++} ++} ++ ++void quantize_row_iq2_k_ref(const float * x, block_iq2_k * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq2_k(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_k(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq2_k * y = (block_iq2_k *)vy; ++ quantize_row_iq2_k_ref(x, y, k); ++} ++ ++size_t quantize_iq2_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ QHelper helper(imatrix, user_data, n_per_row, 16); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_K, n_per_row); ++ helper.quantize(nrows, src, dst, row_size, quantize_row_iq2_k_impl); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq2_k(const block_iq2_k * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ const uint8_t * qs = x[i].qs; ++ ++ uint16_t extra = x[i].extra; ++ ++ int shift = 0; ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ float dl1 = d * ((x[i].scales[ib32] & 0xf) - 8); ++ float dl2 = d * ((x[i].scales[ib32] >> 4) - 8); ++ const int8_t * values1 = extra & 1 ? iq2nl_values + 4 : iq2nl_values; ++ const int8_t * values2 = extra & 2 ? iq2nl_values + 4 : iq2nl_values; ++ extra >>= 2; ++ for (int j = 0; j < 16; ++j) { ++ y[j+ 0] = dl1 * values1[(qs[j+ 0] >> shift) & 3]; ++ y[j+16] = dl2 * values2[(qs[j+16] >> shift) & 3]; ++ } ++ y += 32; ++ shift += 2; ++ if (shift == 8) { qs += 32; shift = 0; } ++ } ++ ++ } ++ ++} ++ ++void vec_dot_iq2_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ GGML_ABORT("not implemented"); ++ ++} ++ ++namespace { ++#if defined(__AVX2__) ++inline void to_values_i32(__m256i idx, __m256i ivalues, __m256i * iv) { ++ auto ival = _mm256_shuffle_epi8(ivalues, idx); ++ auto ival_1 = _mm256_srli_si256(ival, 8); ++ iv[0] = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(ival)); ++ iv[1] = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(ival_1)); ++ iv[2] = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(ival, 1)); ++ iv[3] = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(ival_1, 1)); ++} ++inline __m256i to_int8(const __m256i * ibest) { ++ auto i0 = _mm256_packs_epi32(ibest[0], ibest[1]); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15 ++ auto i1 = _mm256_packs_epi32(ibest[2], ibest[3]); // 16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31 ++ auto idx = _mm256_packs_epi16(i0, i1); // 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 ++ auto idx_l = _mm256_castsi256_si128(idx); ++ auto idx_h = _mm256_extracti128_si256(idx, 1); ++ auto idx1 = _mm_unpacklo_epi32(idx_l, idx_h); // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ++ auto idx2 = _mm_unpackhi_epi32(idx_l, idx_h); ++ return MM256_SET_M128I(idx2, idx1); ++} ++bool compute_1block_iq2ks(float d, const __m256 * vx, const __m256 * vw, const int8_t * values, __m256i & this_idx, float & best_d, float & score) { ++ constexpr int kBlockSize = 32; ++ uint32_t aux32; ++ std::memcpy(&aux32, values, sizeof(aux32)); ++ auto ivalues = _mm256_set1_epi32(aux32); ++ __m256 vbest[8]; ++ __m256i ibest[8]; ++ auto val = _mm256_set1_ps(d*values[0]); ++ auto ival = _mm256_set1_epi32(0); ++ for (int k = 0; k < kBlockSize/8; ++k) { ++ auto diff = _mm256_sub_ps(vx[k], val); ++ vbest[k] = _mm256_mul_ps(diff, diff); ++ ibest[k] = ival; ++ diff = _mm256_add_ps(vx[k], val); ++ vbest[k+4] = _mm256_mul_ps(diff, diff); ++ ibest[k+4] = ival; ++ } ++ for (int j = 1; j < 4; ++j) { ++ val = _mm256_set1_ps(d*values[j]); ++ ival = _mm256_set1_epi32(j); ++ for (int k = 0; k < kBlockSize/8; ++k) { ++ auto diff = _mm256_sub_ps(vx[k], val); ++ diff = _mm256_mul_ps(diff, diff); ++ auto mask = _mm256_cmp_ps(diff, vbest[k], _CMP_LT_OQ); ++ vbest[k] = _mm256_or_ps(_mm256_and_ps(mask, diff), _mm256_andnot_ps(mask, vbest[k])); ++ auto imask = _mm256_castps_si256(mask); ++ ibest[k] = _mm256_or_si256(_mm256_and_si256(imask, ival), _mm256_andnot_si256(imask, ibest[k])); ++ diff = _mm256_add_ps(vx[k], val); ++ diff = _mm256_mul_ps(diff, diff); ++ mask = _mm256_cmp_ps(diff, vbest[k+4], _CMP_LT_OQ); ++ vbest[k+4] = _mm256_or_ps(_mm256_and_ps(mask, diff), _mm256_andnot_ps(mask, vbest[k+4])); ++ imask = _mm256_castps_si256(mask); ++ ibest[k+4] = _mm256_or_si256(_mm256_and_si256(imask, ival), _mm256_andnot_si256(imask, ibest[k+4])); ++ } ++ } ++ bool result = false; ++ auto idx1 = to_int8(ibest+0); ++ auto idx2 = to_int8(ibest+4); ++ to_values_i32(idx1, ivalues, ibest+0); ++ to_values_i32(idx2, ivalues, ibest+4); ++ auto vsqx_1 = _mm256_setzero_ps(); ++ auto vsq2_1 = _mm256_setzero_ps(); ++ auto vsqx_2 = _mm256_setzero_ps(); ++ auto vsq2_2 = _mm256_setzero_ps(); ++ for (int k = 0; k < 4; ++k) { ++ auto vq1 = _mm256_cvtepi32_ps(ibest[k+0]); ++ auto vwq1 = _mm256_mul_ps(vw[k], vq1); ++ auto vq2 = _mm256_cvtepi32_ps(ibest[k+4]); ++ auto vwq2 = _mm256_mul_ps(vw[k], vq2); ++ vsqx_1 = _mm256_fmadd_ps(vwq1, vx[k], vsqx_1); ++ vsq2_1 = _mm256_fmadd_ps(vwq1, vq1, vsq2_1); ++ vsqx_2 = _mm256_fmadd_ps(vwq2, vx[k], vsqx_2); ++ vsq2_2 = _mm256_fmadd_ps(vwq2, vq2, vsq2_2); ++ } ++ auto sumqx_1 = hsum_float_8(vsqx_1); ++ auto sumq2_1 = hsum_float_8(vsq2_1); ++ auto sumqx_2 = hsum_float_8(vsqx_2); ++ auto sumq2_2 = hsum_float_8(vsq2_2); ++ if (sumq2_1 > 0) { ++ best_d = sumqx_1/sumq2_1; ++ score = sumqx_1 * best_d; ++ this_idx = idx1; ++ result = true; ++ } ++ if (sumq2_2 > 0 && (!result || sumqx_2*sumqx_2 > score*sumq2_2)) { ++ best_d = sumqx_2/sumq2_2; ++ score = sumqx_2 * best_d; ++ this_idx = idx2; ++ result = true; ++ } ++ return result; ++} ++float compute_1block_iq2ks_rmse(float d, const __m256 * vx, const __m256 * vw, const int8_t * values, __m256i & this_idx) { ++ constexpr int kBlockSize = 32; ++ uint32_t aux32; ++ std::memcpy(&aux32, values, sizeof(aux32)); ++ auto ivalues = _mm256_set1_epi32(aux32); ++ __m256 vbest[4]; ++ __m256i ibest[4]; ++ auto val = _mm256_set1_ps(d*values[0]); ++ auto ival = _mm256_set1_epi32(0); ++ for (int k = 0; k < kBlockSize/8; ++k) { ++ auto diff = _mm256_sub_ps(vx[k], val); ++ vbest[k] = _mm256_mul_ps(diff, diff); ++ ibest[k] = ival; ++ } ++ for (int j = 1; j < 4; ++j) { ++ val = _mm256_set1_ps(d*values[j]); ++ ival = _mm256_set1_epi32(j); ++ for (int k = 0; k < kBlockSize/8; ++k) { ++ auto diff = _mm256_sub_ps(vx[k], val); ++ diff = _mm256_mul_ps(diff, diff); ++ auto mask = _mm256_cmp_ps(diff, vbest[k], _CMP_LT_OQ); ++ vbest[k] = _mm256_or_ps(_mm256_and_ps(mask, diff), _mm256_andnot_ps(mask, vbest[k])); ++ auto imask = _mm256_castps_si256(mask); ++ ibest[k] = _mm256_or_si256(_mm256_and_si256(imask, ival), _mm256_andnot_si256(imask, ibest[k])); ++ } ++ } ++ auto idx = to_int8(ibest); ++ to_values_i32(idx, ivalues, ibest); ++ auto vd = _mm256_set1_ps(-d); ++ auto vrmse = _mm256_setzero_ps(); ++ for (int k = 0; k < 4; ++k) { ++ auto vq = _mm256_cvtepi32_ps(ibest[k]); ++ auto diff = _mm256_fmadd_ps(vd, vq, vx[k]); ++ auto wdiff = _mm256_mul_ps(vw[k], diff); ++ vrmse = _mm256_fmadd_ps(wdiff, diff, vrmse); ++ } ++ this_idx = idx; ++ return hsum_float_8(vrmse); ++} ++void quantize_row_iq2_ks_fast_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_sw, int8_t * all_Ls) { ++ ++ constexpr int kBlockSize = 32; ++ ++ ggml_half * dptr = (ggml_half *)vy; ++ *dptr = GGML_FP32_TO_FP16(0.f); ++ ++ block_iq2_ks * y = (block_iq2_ks *)(dptr + 1); ++ ++ float weight[kBlockSize]; ++ ++ const int8_t * shifted_values = iq2nl_values + 4; ++ ++ const int nblock = n_per_row/QK_K; ++ ++ __m256 vx[4], vw[4]; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq2_ks)); ++ ++ auto scales = all_scales + ibl*(QK_K/kBlockSize); ++ auto sw = all_sw + ibl*(QK_K/kBlockSize); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ ++ uint16_t extra = 0; ++ ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0, sumw = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ sumw += weight[j]; ++ } ++ sw[ib] = sumw; ++ if (amax < 1e-14f) { ++ scales[ib] = 0; ++ continue; ++ } ++ for (int k = 0; k < 4; ++k) { ++ vx[k] = _mm256_loadu_ps(xb + 8*k); ++ vw[k] = _mm256_loadu_ps(weight + 8*k); ++ } ++ float d = max/iq2nl_values[7]; ++ float best = 0; ++ __m256i this_idx; ++ float this_d, this_score; ++ if (compute_1block_iq2ks(d, vx, vw, iq2nl_values, this_idx, this_d, this_score)) { ++ best = this_score; d = this_d; ++ } ++ for (int itry = -13; itry <= 13; ++itry) { ++ if (compute_1block_iq2ks(max/(iq2nl_values[0] + 0.5f*itry), vx, vw, iq2nl_values, this_idx, this_d, this_score)) { ++ if (this_score > best) { ++ best = this_score; d = this_d; ++ } ++ } ++ } ++ bool is_shifted = false; ++ for (int itry = -13; itry <= 13; ++itry) { ++ if (compute_1block_iq2ks(max/(iq2nl_values[4] + 0.5f*itry), vx, vw, iq2nl_values + 4, this_idx, this_d, this_score)) { ++ if (this_score > best) { ++ best = this_score; d = this_d; is_shifted = true; ++ } ++ } ++ } ++ scales[ib] = d; ++ if (is_shifted) extra |= (1 << ib); ++ } ++ y[ibl].extra = extra; ++ } ++ ++ float d = make_qx_quants(nblock*(QK_K/kBlockSize), 16, all_scales, all_Ls, all_sw); ++ ++ if (!d) return; ++ ++ auto vsumqx = _mm256_setzero_ps(); ++ auto vsumq2 = _mm256_setzero_ps(); ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto scales = all_scales + ibl*(QK_K/kBlockSize); ++ auto xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ auto Ls = all_Ls + ibl*(QK_K/kBlockSize); ++ __m256i idx[4]; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq2nl_values; ++ uint32_t aux32; ++ std::memcpy(&aux32, block_values, sizeof(aux32)); ++ auto ivalues = _mm256_set1_epi32(aux32); ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ for (int k = 0; k < 4; ++k) { ++ vx[k] = _mm256_loadu_ps(xb + 8*k); ++ vw[k] = _mm256_loadu_ps(weight + 8*k); ++ } ++ int ls = Ls[ib] - 16; ++ float dl = d*ls; ++ __m256i idx1, idx2; ++ auto rmse1 = compute_1block_iq2ks_rmse(dl, vx, vw, block_values, idx1); ++ if (Ls[ib] > 0 && dl > scales[ib]) { ++ auto rmse2 = compute_1block_iq2ks_rmse(d*(Ls[ib] - 17), vx, vw, block_values, idx2); ++ if (rmse2 < rmse1) { ++ --Ls[ib]; idx1 = idx2; ++ } ++ } ++ else if (Ls[ib] < 15 && dl < scales[ib]) { ++ auto rmse2 = compute_1block_iq2ks_rmse(d*(Ls[ib] - 15), vx, vw, block_values, idx2); ++ if (rmse2 < rmse1) { ++ ++Ls[ib]; idx1 = idx2; ++ } ++ } ++ __m256i iv[4]; ++ to_values_i32(idx1, ivalues, iv); ++ auto vd = _mm256_set1_ps(Ls[ib] - 16); ++ for (int k = 0; k < 4; ++k) { ++ auto vq = _mm256_mul_ps(vd, _mm256_cvtepi32_ps(iv[k])); ++ auto wvq = _mm256_mul_ps(vw[k], vq); ++ vsumqx = _mm256_fmadd_ps(wvq, vx[k], vsumqx); ++ vsumq2 = _mm256_fmadd_ps(wvq, vq, vsumq2); ++ } ++ ls = Ls[ib]; ++ y[ibl].scales[ib/2] |= ((ls & 0xf) << 4*(ib%2)); ++ y[ibl].extra |= ((ls >> 4) << (8 + ib)); ++ idx[ib % 4] = idx1; ++ if ((ib % 4) == 3) { ++ auto vqs1 = _mm256_or_si256(idx[0], _mm256_slli_epi16(idx[1], 2)); ++ auto vqs2 = _mm256_or_si256(_mm256_slli_epi16(idx[2], 4), _mm256_slli_epi16(idx[3], 6)); ++ auto vqs = _mm256_or_si256(vqs1, vqs2); ++ _mm256_storeu_si256((__m256i *)y[ibl].qs + ib/4, vqs); ++ } ++ } ++ } ++ float sumqx = hsum_float_8(vsumqx); ++ float sumq2 = hsum_float_8(vsumq2); ++ *dptr = GGML_FP32_TO_FP16(1.000f*(sumq2 > 0 ? sumqx/sumq2 : d)); ++} ++#endif ++void quantize_row_iq2_ks_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_sw, int8_t * all_Ls) { ++ ++ constexpr int kBlockSize = 32; ++ constexpr int kMax_i1 = 3*kBlockSize/4; ++ constexpr int kMin_i3 = kBlockSize/4; ++ ++ ggml_half * dptr = (ggml_half *)vy; ++ *dptr = GGML_FP32_TO_FP16(0.f); ++ ++ block_iq2_ks * y = (block_iq2_ks *)(dptr + 1); ++ ++ float weight[kBlockSize]; ++ float sumx[kBlockSize+1], sumw[kBlockSize+1]; ++ ++ std::array, kBlockSize> pairs; ++ ++ float val [4] = {float(iq2nl_values[0]), float(iq2nl_values[1]), float(iq2nl_values[2]), float(iq2nl_values[3])}; ++ float sval[4] = {float(iq2nl_values[4]), float(iq2nl_values[5]), float(iq2nl_values[6]), float(iq2nl_values[7])}; ++ ++ const int8_t * shifted_values = iq2nl_values + 4; ++ ++ const int nblock = n_per_row/QK_K; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq2_ks)); ++ ++ auto scales = all_scales + ibl*(QK_K/kBlockSize); ++ auto sw = all_sw + ibl*(QK_K/kBlockSize); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ ++ uint16_t extra = 0; ++ ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ sw[ib] = 0; ++ float amax = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ sw[ib] += weight[j]; ++ pairs[j] = {xb[j], j}; ++ float ax = std::abs(xb[j]); ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ std::sort(pairs.begin(), pairs.end()); ++ sumx[0] = sumw[0] = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ int jj = pairs[j].second; ++ sumw[j+1] = sumw[j] + weight[jj]; ++ sumx[j+1] = sumx[j] + weight[jj]*xb[jj]; ++ } ++ float best = 0, d = 0; ++ bool is_shifted = false; ++ float sumqx, sumq2; ++ for (int i1 = 0; i1 < kMax_i1; ++i1) { ++ for (int i2 = i1; i2 < kBlockSize; ++i2) { ++ for (int i3 = std::max(i2, kMin_i3); i3 < kBlockSize; ++i3) { ++ sumqx = (sumx[i1] - sumx[ 0])*val[0] + (sumx[i2] - sumx[i1])*val[1] ++ + (sumx[i3] - sumx[i2])*val[2] + (sumx[kBlockSize] - sumx[i3])*val[3]; ++ sumq2 = (sumw[i1] - sumw[ 0])*val[0]*val[0] + (sumw[i2] - sumw[i1])*val[1]*val[1] ++ + (sumw[i3] - sumw[i2])*val[2]*val[2] + (sumw[kBlockSize] - sumw[i3])*val[3]*val[3]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = false; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*sval[0] + (sumx[i2] - sumx[i1])*sval[1] ++ + (sumx[i3] - sumx[i2])*sval[2] + (sumx[kBlockSize] - sumx[i3])*sval[3]; ++ sumq2 = (sumw[i1] - sumw[ 0])*sval[0]*sval[0] + (sumw[i2] - sumw[i1])*sval[1]*sval[1] ++ + (sumw[i3] - sumw[i2])*sval[2]*sval[2] + (sumw[kBlockSize] - sumw[i3])*sval[3]*sval[3]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = true; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*val[3] + (sumx[i2 ] - sumx[i1])*val[2] ++ + (sumx[i3] - sumx[i2])*val[1] + (sumx[kBlockSize] - sumx[i3])*val[0]; ++ sumq2 = (sumw[i1] - sumw[ 0])*val[3]*val[3] + (sumw[i2 ] - sumw[i1])*val[2]*val[2] ++ + (sumw[i3] - sumw[i2])*val[1]*val[1] + (sumw[kBlockSize] - sumw[i3])*val[0]*val[0]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = false; ++ } ++ sumqx = (sumx[i1] - sumx[ 0])*sval[3] + (sumx[i2 ] - sumx[i1])*sval[2] ++ + (sumx[i3] - sumx[i2])*sval[1] + (sumx[kBlockSize] - sumx[i3])*sval[0]; ++ sumq2 = (sumw[i1] - sumw[ 0])*sval[3]*sval[3] + (sumw[i2 ] - sumw[i1])*sval[2]*sval[2] ++ + (sumw[i3] - sumw[i2])*sval[1]*sval[1] + (sumw[kBlockSize] - sumw[i3])*sval[0]*sval[0]; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; is_shifted = true; ++ } ++ } ++ } ++ } ++ scales[ib] = d; ++ if (is_shifted) extra |= (1 << ib); ++ } ++ y[ibl].extra = extra; ++ } ++ ++ float d = make_qx_quants(nblock*(QK_K/kBlockSize), 16, all_scales, all_Ls, all_sw); ++ ++ if (!d) return; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ auto Ls = all_Ls + ibl*(QK_K/kBlockSize); ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ int ls = Ls[ib]; ++ y[ibl].scales[ib/2] |= ((ls & 0xf) << 4*(ib%2)); ++ y[ibl].extra |= ((ls >> 4) << (8 + ib)); ++ ls -= 16; ++ float dl = d * ls; ++ if (dl) { ++ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq2nl_values; ++ const float * xb = xbl + kBlockSize*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float idl = 1/dl; ++ uint8_t * qs = y[ibl].qs + 32*(ib/4); ++ for (int j = 0; j < 32; ++j) { ++ const float al = idl*xb[j]; ++ int ibest = best_index_iq2nl(block_values, al); ++ qs[j] |= (ibest << 2*(ib%4)); ++ float w = weight[j]; ++ float q = block_values[ibest]*ls; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ } ++ *dptr = GGML_FP32_TO_FP16(1.030f*(sumq2 > 0 ? sumqx/sumq2 : d)); ++} ++} ++ ++void quantize_row_iq2_ks_ref(const float * x, block_iq2_ks * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq2_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_ks(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq2_ks * y = (block_iq2_ks *)vy; ++ quantize_row_iq2_ks_ref(x, y, k); ++} ++ ++size_t quantize_iq2_ks(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_KS, n_per_row); ++ int nblock = n_per_row/QK_K; ++ std::vector all_scales(nblock*(QK_K/kBlockSize)), all_sw(nblock*(QK_K/kBlockSize)); ++ std::vector all_Ls(nblock*(QK_K/kBlockSize)); ++ auto q_func = [&all_scales, &all_sw, &all_Ls] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++#ifdef __AVX2__ ++ if (user_data && !user_data->slow_iq2_ks) { ++ quantize_row_iq2_ks_fast_impl(x, vy, n_per_row, imatrix, all_scales.data(), all_sw.data(), all_Ls.data()); ++ return; ++ } ++#endif ++ quantize_row_iq2_ks_impl(x, vy, n_per_row, imatrix, all_scales.data(), all_sw.data(), all_Ls.data()); ++ }; ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq2_ks(const block_iq2_ks * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ const ggml_half * dptr = (const ggml_half *)x; ++ const float d = GGML_FP16_TO_FP32(*dptr); ++ x = (const block_iq2_ks *)(dptr + 1); ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const uint8_t * qs = x[i].qs; ++ ++ uint16_t extra = x[i].extra; ++ ++ int shift = 0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ float dl1 = d * (((x[i].scales[ib64] & 0xf) | ((extra >> 4) & 0x10)) - 16); ++ float dl2 = d * (((x[i].scales[ib64] >> 4) | ((extra >> 5) & 0x10)) - 16); ++ const int8_t * values1 = extra & 1 ? iq2nl_values + 4 : iq2nl_values; ++ const int8_t * values2 = extra & 2 ? iq2nl_values + 4 : iq2nl_values; ++ extra >>= 2; ++ for (int j = 0; j < 32; ++j) { ++ y[j+ 0] = dl1 * values1[(qs[j] >> (shift+0)) & 3]; ++ y[j+32] = dl2 * values2[(qs[j] >> (shift+2)) & 3]; ++ } ++ y += 64; ++ shift += 4; ++ if (shift == 8) { qs += 32; shift = 0; } ++ } ++ ++ } ++ ++} ++ ++void vec_dot_iq2_ks_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_KS, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ const ggml_half * dptr = (const ggml_half *)vx; ++ const float d = GGML_FP16_TO_FP32(*dptr); ++ const block_iq2_ks * x = (const block_iq2_ks *)(dptr + 1); ++ const block_q8_K * y = (const block_q8_K *)vy; ++ ++ const int nb = n / QK_K; ++ float sumf = 0; ++ for (int i = 0; i < nb; i++) { ++ const uint8_t * qs = x[i].qs; ++ const int8_t * q8 = y[i].qs; ++ uint16_t extra = x[i].extra; ++ int sumi = 0; ++ for (int ib128 = 0; ib128 < QK_K/128; ++ib128) { ++ int d1 = (((x[i].scales[2*ib128+0] & 0xf) | ((extra >> 4) & 0x10)) - 16); ++ int d2 = (((x[i].scales[2*ib128+0] >> 4) | ((extra >> 5) & 0x10)) - 16); ++ int d3 = (((x[i].scales[2*ib128+1] & 0xf) | ((extra >> 6) & 0x10)) - 16); ++ int d4 = (((x[i].scales[2*ib128+1] >> 4) | ((extra >> 7) & 0x10)) - 16); ++ const int8_t * values1 = extra & 1 ? iq2nl_values + 4 : iq2nl_values; ++ const int8_t * values2 = extra & 2 ? iq2nl_values + 4 : iq2nl_values; ++ const int8_t * values3 = extra & 4 ? iq2nl_values + 4 : iq2nl_values; ++ const int8_t * values4 = extra & 8 ? iq2nl_values + 4 : iq2nl_values; ++ extra >>= 4; ++ int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; ++ for (int j = 0; j < 32; ++j) { ++ sumi1 += q8[j+ 0] * values1[(qs[j] >> 0) & 3]; ++ sumi2 += q8[j+32] * values2[(qs[j] >> 2) & 3]; ++ sumi3 += q8[j+64] * values3[(qs[j] >> 4) & 3]; ++ sumi4 += q8[j+96] * values4[(qs[j] >> 6) & 3]; ++ } ++ sumi += d1*sumi1 + d2*sumi2 + d3*sumi3 + d4*sumi4; ++ q8 += 128; ++ qs += 32; ++ } ++ sumf += y[i].d * sumi; ++ } ++ ++ *s = d * sumf; ++ ++} ++ ++// ++// ======================================== iq2_kl ++// ++namespace { ++ ++const int8_t iq3nl_index[111] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, ++ 9, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 11, 11, 4, 4, 4, 4, ++ 4, 4, 4, 4, 4, 4, 12, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 13, 13, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ++ 6, 6, 6, 6, 14, 14, 7, 7, 7, 7, 7, 7, 7, 7, 7 ++}; ++inline int best_index_iq3nl(const int8_t * values, float x) { ++ int ix = (int)x - values[0]; ++ if (ix < 0 || ix >= 111) return ix < 0 ? 0 : 7; ++ ix = iq3nl_index[ix]; ++ return ix < 8 ? ix : x - values[ix-8] < values[ix-7] - x ? ix-8 : ix-7; ++} ++ ++void quantize_row_iq2_kl_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales) { ++ constexpr int kBlockSize = 32; ++ constexpr float kSigmaFactor = 2.25f; ++ constexpr int ntry = 5; ++ static const int k_index[64] = {-1, -2, 0, -3, -4, 1, -5, -6, 2, -7, -8, 3, -9, 4, -10, 5, -11, 6, 7, -12, 8, 9, 10, -13, 11, -14, -15, -16, 12, 13, -17, ++ 14, -18, -19, 15, 16, 17, 18, 19, -20, -21, 20, 21, 22, 23, 24, -22, -23, 25, -24, 26, -25, 27, -26, 28, 29, -27, -28, 30, -29, -30, 31, -31, -32}; ++ static const std::vector> k_neighbours = { ++ { 2, 0, 6, 11, 7, 3, 8, 15, }, ++ { 0, 2, 3, 6, 7, 1, 8, 4, }, ++ { 0, 1, 3, 4, 8, 7, 9, 6, }, ++ { 1, 0, 3, 4, 8, 9, 7, 10, }, ++ { 1, 4, 5, 10, 9, 3, 8, 0, }, ++ { 5, 1, 4, 10, 9, 14, 8, 3, }, ++ { 6, 2, 7, 0, 3, 11, 8, 15, }, ++ { 3, 7, 0, 6, 8, 4, 12, 9, }, ++ { 3, 4, 8, 9, 1, 7, 12, 10, }, ++ { 4, 10, 5, 9, 1, 8, 13, 14, }, ++ { 11, 2, 6, 7, 20, 15, 25, 21, }, ++ { 8, 7, 3, 12, 9, 16, 17, 13, }, ++ { 14, 5, 10, 19, 9, 13, 4, 18, }, ++ { 6, 15, 7, 11, 20, 21, 16, 2, }, ++ { 15, 7, 16, 6, 21, 12, 17, 22, }, ++ { 12, 16, 17, 8, 15, 7, 13, 22, }, ++ { 19, 10, 13, 18, 14, 9, 12, 24, }, ++ { 11, 20, 25, 6, 15, 2, 21, 7, }, ++ { 20, 15, 21, 6, 11, 7, 16, 26, }, ++ { 14, 19, 29, 10, 28, 18, 13, 24, }, ++ { 25, 11, 20, 21, 15, 6, 26, 30, }, ++ { 19, 24, 28, 18, 29, 23, 13, 17, }, ++ { 29, 19, 14, 28, 24, 18, 10, 13, }, ++ { 20, 26, 21, 25, 30, 15, 22, 16, }, ++ { 27, 26, 22, 23, 21, 30, 16, 24, }, ++ { 27, 24, 28, 31, 23, 18, 22, 17, }, ++ { 25, 30, 20, 26, 21, 11, 15, 22, }, ++ { 30, 26, 25, 20, 21, 27, 22, 15, }, ++ { 30, 27, 31, 26, 22, 23, 21, 24, }, ++ { 31, 27, 30, 26, 28, 23, 22, 24, }, ++ { 31, 28, 29, 27, 24, 23, 19, 18, }, ++ { 29, 28, 31, 24, 19, 27, 14, 18, }, ++ }; ++ auto values = iq3nl_values; ++ std::pair grid[32]; ++ for (int j = 0; j < 64; ++j) { ++ if (int i = k_index[j]; i >= 0) { ++ int i1 = j/8, i2 = j%8; ++ grid[i] = {values[i1], values[i2]}; ++ } ++ } ++ ++ ggml_half * dptr = (ggml_half *)vy; ++ auto y = (block_iq2_kl *)(dptr + 1); ++ ++ float weight[kBlockSize]; ++ ++ auto index = [&grid, values] (float id, float x1, float x2, float w1, float w2) { ++ float sx1 = id*x1; ++ float sx2 = id*x2; ++ int l1 = best_index_iq3nl(values, sx1); ++ int l2 = best_index_iq3nl(values, sx2); ++ int i = k_index[8*l1 + l2]; ++ if (i >= 0) return i; ++ auto& neigh = k_neighbours[-i-1]; ++ float best = std::numeric_limits::max(); ++ int ibest = -1; ++ for (auto& n : neigh) { ++ float diff1 = grid[n].first - sx1; ++ float diff2 = grid[n].second - sx2; ++ float score = w1*diff1*diff1 + w2*diff2*diff2; ++ if (score < best) { ++ best = score; ibest = n; ++ } ++ } ++ GGML_ASSERT(ibest >= 0); ++ return ibest; ++ }; ++ ++ float max_scale = 0, max_abs_scale = 0; ++ ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ std::memset(&y[ibl], 0, sizeof(block_iq2_kl)); ++ auto scales = all_scales + ibl*(QK_K/kBlockSize); ++ auto xbl = x + ibl*QK_K; ++ float sigma2 = 0; ++ for (int j = 0; j < QK_K; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= kSigmaFactor/QK_K; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ auto xb = xbl + ib*kBlockSize; ++ if (quant_weights) { ++ auto qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j]*sqrt(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = std::abs(xb[j]); //xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ float ax = std::abs(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < kBlockSize; j += 2) { ++ float w1 = weight[j+0]; ++ float w2 = weight[j+1]; ++ int idx = index(id, xb[j+0], xb[j+1], w1, w2); ++ float q1 = grid[idx].first ; ++ float q2 = grid[idx].second; ++ sumqx_p += w1*q1*xb[j] + w2*q2*xb[j+1]; ++ sumq2_p += w1*q1*q1 + w2*q2*q2; ++ idx = index(-id, xb[j+0], xb[j+1], w1, w2); ++ q1 = grid[idx].first ; ++ q2 = grid[idx].second; ++ sumqx_m += w1*q1*xb[j] + w2*q2*xb[j+1]; ++ sumq2_m += w1*q1*q1 + w2*q2*q2; ++ } ++ d = sumqx_p/sumq2_p; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < kBlockSize; j += 2) { ++ float w1 = weight[j+0]; ++ float w2 = weight[j+1]; ++ int idx = index(id, xb[j+0], xb[j+1], w1, w2); ++ float q1 = grid[idx].first ; ++ float q2 = grid[idx].second; ++ sumqx_p += w1*q1*xb[j] + w2*q2*xb[j+1]; ++ sumq2_p += w1*q1*q1 + w2*q2*q2; ++ idx = index(-id, xb[j+0], xb[j+1], w1, w2); ++ q1 = grid[idx].first ; ++ q2 = grid[idx].second; ++ sumqx_m += w1*q1*xb[j] + w2*q2*xb[j+1]; ++ sumq2_m += w1*q1*q1 + w2*q2*q2; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; ++ } ++ } ++ scales[ib] = d; ++ float ad = std::abs(d); ++ if (ad > max_abs_scale) { ++ max_abs_scale = ad; max_scale = d; ++ } ++ } ++ } ++ ++ if (!max_abs_scale) { ++ dptr[0] = GGML_FP32_TO_FP16(0.f); ++ return; ++ } ++ ++ float d = -max_scale/32; ++ float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ auto scales = all_scales + ibl*(QK_K/kBlockSize); ++ auto xbl = x + ibl*QK_K; ++ float sigma2 = 0; ++ for (int j = 0; j < QK_K; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= kSigmaFactor/QK_K; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ auto xb = xbl + ib*kBlockSize; ++ if (quant_weights) { ++ auto qw = quant_weights + ibl*QK_K + ib*kBlockSize; ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j]*sqrt(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = std::abs(xb[j]); //xb[j]*xb[j]; ++ } ++ int ls = nearest_int(id*scales[ib]); ++ ls = std::max(-32, std::min(31, ls)); ++ int lsmin = std::max(-32, ls-1); ++ int lsmax = std::min( 31, ls+1); ++ float best_score = std::numeric_limits::max(); ++ int best_ls = ls; ++ for (int ils = lsmin; ils <= lsmax; ++ils) { ++ float dl = d*ils; ++ float idl = dl ? 1/dl : 0.f; ++ float score = 0; ++ for (int j = 0; j < kBlockSize/2; ++j) { ++ float w1 = weight[2*j+0]; ++ float w2 = weight[2*j+1]; ++ int idx = index(idl, xb[2*j+0], xb[2*j+1], w1, w2); ++ float diff1 = dl*grid[idx].first - xb[2*j+0]; ++ float diff2 = dl*grid[idx].second - xb[2*j+1]; ++ score += w1*diff1*diff1 + w2*diff2*diff2; ++ } ++ if (score < best_score) { ++ best_score = score; ++ best_ls = ils; ++ } ++ } ++ ls = best_ls; ++ int uls = ls + 32; ++ y[ibl].scales_l[ib%4] |= ((uls & 0xf) << 4*(ib/4)); ++ y[ibl].scales_h |= ((uls >> 4) << 2*ib); ++ if (ls == 0) continue; ++ float dl = d*ls; ++ float idl = 1/dl; ++ for (int j = 0; j < kBlockSize/2; ++j) { ++ float w1 = weight[2*j+0]; ++ float w2 = weight[2*j+1]; ++ int idx = index(idl, xb[2*j+0], xb[2*j+1], w1, w2); ++ y[ibl].qs[16*(ib/2) + j] |= ((idx & 0xf) << 4*(ib%2)); ++ y[ibl].qh[j] |= ((idx >> 4) << ib); ++ float q1 = ls*grid[idx].first ; ++ float q2 = ls*grid[idx].second; ++ sumqx += w1*q1*xb[2*j] + w2*q2*xb[2*j+1]; ++ sumq2 += w1*q1*q1 + w2*q2*q2; ++ } ++ } ++ } ++ if (sumq2 > 0) d = sumqx/sumq2; ++ ++ dptr[0] = GGML_FP32_TO_FP16(1.025f * d); ++ ++} ++} ++ ++void quantize_row_iq2_kl_ref(const float * x, block_iq2_kl * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq2_kl(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_kl(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq2_kl * y = (block_iq2_kl *)vy; ++ quantize_row_iq2_kl_ref(x, y, k); ++} ++ ++size_t quantize_iq2_kl(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_KL, n_per_row); ++ int nblock = n_per_row/QK_K; ++ std::vector all_scales(nblock*(QK_K/kBlockSize)); ++ auto q_func = [&all_scales] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq2_kl_impl(x, vy, n_per_row, imatrix, all_scales.data()); ++ }; ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq2_kl(const block_iq2_kl * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ const ggml_half * dptr = (const ggml_half *)x; ++ const float d = GGML_FP16_TO_FP32(*dptr); ++ x = (const block_iq2_kl *)(dptr + 1); ++ ++ for (int i = 0; i < nb; i++) { ++ ++ auto qs = x[i].qs; ++ auto qh = x[i].qh; ++ auto scales_h = x[i].scales_h; ++ ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ float dl1 = d * (int(((x[i].scales_l[(2*ib64+0)%4] >> 4*(ib64/2)) & 0xf) | (((scales_h >> (4*ib64+0)) & 3) << 4)) - 32); ++ float dl2 = d * (int(((x[i].scales_l[(2*ib64+1)%4] >> 4*(ib64/2)) & 0xf) | (((scales_h >> (4*ib64+2)) & 3) << 4)) - 32); ++ for (int j = 0; j < 16; ++j) { ++ const int8_t * val1 = (const int8_t *)(iq2kl_values + ((qs[j] & 0xf) | (((qh[j] >> (2*ib64+0)) & 1) << 4))); ++ const int8_t * val2 = (const int8_t *)(iq2kl_values + ((qs[j] >> 4) | (((qh[j] >> (2*ib64+1)) & 1) << 4))); ++ y[2*j+ 0] = dl1 * val1[0]; ++ y[2*j+ 1] = dl1 * val1[1]; ++ y[2*j+32] = dl2 * val2[0]; ++ y[2*j+33] = dl2 * val2[1]; ++ } ++ y += 64; ++ qs += 16; ++ } ++ ++ } ++} ++ ++void vec_dot_iq2_kl_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_KL, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++} ++ ++// ++// ============================================== iq3_k ++// ++namespace { ++ ++static void quantize_row_iq3_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ ++ constexpr int ntry = 3; ++ ++ block_iq3_k * y = (block_iq3_k *)vy; ++ ++ float scales[QK_K/16]; ++ float weight[16]; ++ uint8_t L[16]; ++ ++ const int8_t * shifted_values = iq3nl_values + 8; ++ ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq3_k)); ++ y[ibl].d = GGML_FP32_TO_FP16(0.f); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 1.5f*sumx2/QK_K; ++ ++ uint16_t extra = 0; ++ ++ float max_abs_scale = 0; ++ ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < 16; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/iq3nl_values[0] : max/iq3nl_values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ float best = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(iq3nl_values, al); ++ float q = iq3nl_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(iq3nl_values, -al); ++ q = iq3nl_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0) { ++ d = sumqx_p/sumq2_p; ++ best = d*sumqx_p; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ bool is_shifted = false; ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (2*itry + iq3nl_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(iq3nl_values, al); ++ float q = iq3nl_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(iq3nl_values, -al); ++ q = iq3nl_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (2*itry + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (!d) { ++ scales[ib] = 0; continue; ++ } ++ ++ const int8_t * block_values = is_shifted ? shifted_values : iq3nl_values; ++ float sumqx = 0, sumq2 = 0; ++ id = 1/d; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(block_values, al); ++ L[j] = l; ++ float q = block_values[l]; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ if (sumq2 > 0) d = sumqx/sumq2; ++ ++ float best_d = d; ++ for (int iter = 0; iter < 128; ++iter) { ++ float gmax = 0; ++ int best_j = -1, dir = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float g = d * w * (xb[j] - d*block_values[L[j]]); ++ if (g > 0 && L[j] < 7) { ++ if (g > gmax) { ++ gmax = g; best_j = j; dir = 1; ++ } ++ } ++ else if (g < 0 && L[j] > 0) { ++ if (-g > gmax) { ++ gmax = -g; best_j = j; dir = -1; ++ } ++ } ++ } ++ if (best_j < 0) break; ++ ++ float w = weight[best_j]; ++ sumqx += w*xb[best_j]*(block_values[L[best_j]+dir] - block_values[L[best_j]]); ++ sumq2 += w*(block_values[L[best_j]+dir]*block_values[L[best_j]+dir] - block_values[L[best_j]]*block_values[L[best_j]]); ++ L[best_j] += dir; ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ best_d = sumqx/sumq2; best = best_d*sumqx; ++ } ++ else if (iter > 8) break; ++ ++ } ++ ++ scales[ib] = d; ++ ++ if (is_shifted) extra |= (1 << ib); ++ ++ float abs_scale = fabsf(scales[ib]); ++ max_abs_scale = MAX(max_abs_scale, abs_scale); ++ } ++ ++ if (!max_abs_scale) continue; ++ ++ float d = max_abs_scale/31; ++ y[ibl].extra = extra; ++ float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ int ls = nearest_int(0.5f*(id*fabsf(scales[ib])-1)); ++ ls = MAX(0, MIN(15, ls)); ++ y[ibl].scales_l[ib/2] |= (ls << 4*(ib%2)); ++ if (scales[ib] < 0) y[ibl].scales_h |= (1 << ib); ++ ls = (2*ls + 1) * (scales[ib] < 0 ? -1 : 1); ++ float dl = d * ls; ++ if (dl) { ++ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq3nl_values; ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float idl = 1/dl; ++ int ib32 = ib/2; ++ int offset = 16*(ib%2); ++ uint8_t * qs = y[ibl].qs + 32*(ib32/4) + offset; ++ uint8_t * qh = y[ibl].qh + 32*(ib32/8) + offset; ++ for (int j = 0; j < 16; ++j) { ++ const float al = idl*xb[j]; ++ int ibest = best_index_iq3nl(block_values, al); ++ qs[j] |= ((ibest & 3) << 2*(ib32%4)); ++ qh[j] |= ((ibest >> 2) << (ib32%8)); ++ float w = weight[j]; ++ float q = block_values[ibest]*ls; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ y[ibl].d = GGML_FP32_TO_FP16(1.01f*(sumq2 > 0 ? sumqx/sumq2 : d)); ++ ++ } ++} ++ ++} ++ ++void quantize_row_iq3_k_ref(const float * x, block_iq3_k * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq3_k(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_k(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq3_k * y = (block_iq3_k *)vy; ++ quantize_row_iq3_k_ref(x, y, k); ++} ++ ++size_t quantize_iq3_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ QHelper helper(imatrix, user_data, n_per_row, 16); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ3_K, n_per_row); ++ helper.quantize(nrows, src, dst, row_size, quantize_row_iq3_k_impl); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq3_k(const block_iq3_k * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ const uint8_t * qs = x[i].qs; ++ const uint8_t * qh = x[i].qh; ++ ++ uint16_t sh = x[i].scales_h; ++ uint16_t extra = x[i].extra; ++ ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ float dl1 = d * ((2*(x[i].scales_l[ib32] & 0xf) + 1) * ((sh & 1) ? -1 : 1)); ++ float dl2 = d * ((2*(x[i].scales_l[ib32] >> 4) + 1) * ((sh & 2) ? -1 : 1)); ++ sh >>= 2; ++ const int8_t * values1 = extra & 1 ? iq3nl_values + 8 : iq3nl_values; ++ const int8_t * values2 = extra & 2 ? iq3nl_values + 8 : iq3nl_values; ++ extra >>= 2; ++ int shift_l = 2*(ib32%4); ++ int shift_h = ib32%8; ++ for (int j = 0; j < 16; ++j) { ++ y[j+ 0] = dl1 * values1[((qs[j+ 0] >> shift_l) & 3) | (((qh[j+ 0] >> shift_h) & 1) << 2)]; ++ y[j+16] = dl2 * values2[((qs[j+16] >> shift_l) & 3) | (((qh[j+16] >> shift_h) & 1) << 2)]; ++ } ++ y += 32; ++ if (shift_l == 6) qs += 32; ++ } ++ ++ } ++} ++ ++void vec_dot_iq3_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ GGML_ABORT("not implemented"); ++} ++ ++// ++// ============================================== iq3_ks ++// ++namespace { ++static void quantize_row_iq3_ks_impl(const int super_block_size, const int block_size, ++ int n_per_row, const float * x, char * cy, ++ float * all_scales, float * weight, ++ const int8_t * values, ++ const float * quant_weights, ++ const int ntry) { ++ ++ ggml_half * dptr = (ggml_half *)cy; ++ block_iq3_ks * y = (block_iq3_ks *)(dptr + 1); ++ ++ const int8_t * shifted_values = values + 8; ++ ++ float amax_scale = 0; ++ float max_scale = 0; ++ ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ memset(&y[ibl], 0, sizeof(block_iq3_ks)); ++ const float * xbl = x + ibl*super_block_size; ++ auto scales = all_scales + ibl*(super_block_size/block_size); ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ float best = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0) { ++ d = sumqx_p/sumq2_p; ++ best = d*sumqx_p; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ bool is_shifted = false; ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq3nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq3nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (is_shifted) y[ibl].extra |= (1 << (8 + ib)); ++ scales[ib] = d; ++ float ascale = std::abs(d); ++ if (ascale > amax_scale) { ++ amax_scale = ascale; max_scale = d; ++ } ++ } ++ } ++ float d = -max_scale/16; ++ *dptr = GGML_FP32_TO_FP16(d); ++ if (!d) return; ++ float id = d ? 1/d : 0.f; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ const float * xbl = x + ibl*super_block_size; ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ auto scales = all_scales + (super_block_size/block_size)*ibl; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const int8_t * block_values = (y[ibl].extra >> (8 + ib)) & 0x01 ? shifted_values : values; ++ int l = nearest_int(id*scales[ib]); ++ l = std::max(-16, std::min(15, l)); ++ uint8_t ul = l + 16; ++ y[ibl].scales[ib%4] |= (ul & 0xf) << 4*(ib/4); ++ y[ibl].extra |= (ul >> 4) << ib; ++ float dl = d * l; ++ float idl = dl ? 1/dl : 0.f; ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ auto qs = y[ibl].qs + (ib/4)*block_size; ++ auto qh = y[ibl].qh + (ib/8)*block_size; ++ for (int j = 0; j < block_size; ++j) { ++ uint8_t i = best_index_iq3nl(block_values, idl*xb[j]); ++ qs[j] |= ((i & 3) << 2*(ib%4)); ++ qh[j] |= ((i >> 2) << (ib%8)); ++ float w = weight[j]; ++ float q = block_values[i]*l; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ if (sumq2 > 0) *dptr = GGML_FP32_TO_FP16(sumqx/sumq2); ++} ++} ++ ++void quantize_row_iq3_ks_ref(const float * x, block_iq3_ks * y, int64_t k) { ++ quantize_iq3_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_ks(const float * x, void * y, int64_t k) { ++ quantize_iq3_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++size_t quantize_iq3_ks(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ float weight[kBlockSize]; ++ std::vector all_scales(n_per_row/kBlockSize); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ3_KS, n_per_row); ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ auto q_func = [&all_scales, &weight, block_size = kBlockSize] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq3_ks_impl(QK_K, block_size, n_per_row, x, (char *)vy, all_scales.data(), weight, iq3nl_values, imatrix, 5); ++ }; ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq3_ks(const block_iq3_ks * x, float * y, int64_t k) { ++ constexpr int kBlockSize = 32; ++ static_assert(QK_K/kBlockSize == 8); ++ GGML_ASSERT(k%QK_K == 0); ++ const ggml_half * dptr = (const ggml_half *)x; ++ float d = GGML_FP16_TO_FP32(*dptr); ++ x = (const block_iq3_ks *)(dptr + 1); ++ float dl[8]; ++ int nblock = k/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int j = 0; j < 4; ++j) { ++ int ls1 = (x[ibl].scales[j] & 0xf) | (((x[ibl].extra >> (j+0)) & 1) << 4); ++ int ls2 = (x[ibl].scales[j] >> 4) | (((x[ibl].extra >> (j+4)) & 1) << 4); ++ dl[j+0] = d*(ls1 - 16); ++ dl[j+4] = d*(ls2 - 16); ++ } ++ auto qs = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ for (int i128 = 0; i128 < QK_K/128; ++i128) { ++ for (int ib = 0; ib < 4; ++ib) { ++ const int8_t * values = iq3nl_values + ((x[ibl].extra >> (8 + (4*i128+ib)) & 1) << 3); ++ for (int j = 0; j < kBlockSize; ++j) { ++ y[j] = dl[4*i128 + ib] * values[((qs[j] >> 2*ib) & 3) | (((qh[j] >> (4*i128+ib)) & 1) << 2)]; ++ } ++ y += kBlockSize; ++ } ++ qs += kBlockSize; ++ } ++ } ++} ++ ++void vec_dot_iq3_ks_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_KS, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_ABORT("Not implemented"); ++} ++ ++// ++// ============================================== iq4_K ++// ++void dequantize_row_iq4_k(const block_iq4_k * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const uint8_t * qs = x[i].qs; ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ ++ uint16_t extra = x[i].extra; ++ ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ const uint8_t sh = x[i].scales_h[ib/2] >> 4*(ib%2); ++ const float dl1 = d * (((x[i].scales_l[ib] & 0xf) | ((sh << 4) & 0x30)) - 32); ++ const float dl2 = d * (((x[i].scales_l[ib] >> 4) | ((sh << 2) & 0x30)) - 32); ++ const int8_t * values1 = extra & 1 ? iq4k_values + 16 : iq4k_values; ++ const int8_t * values2 = extra & 2 ? iq4k_values + 16 : iq4k_values; ++ extra >>= 2; ++ for (int j = 0; j < 16; ++j) { ++ y[j+ 0] = dl1 * values1[qs[j] & 0xf]; ++ y[j+16] = dl2 * values2[qs[j] >> 4]; ++ } ++ y += 32; ++ qs += 16; ++ } ++ } ++} ++ ++void vec_dot_iq4_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ const int nb = n / QK_K; ++ ++ const block_iq4_k * x = (const block_iq4_k *)vx; ++ const block_q8_K * y = (const block_q8_K *)vy; ++ ++ float sumf = 0; ++ for (int ibl = 0; ibl < nb; ++ibl) { ++ const float d4d8 = GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d; ++ uint16_t extra = x[ibl].extra; ++ uint32_t h = *((const uint32_t *)x[ibl].scales_h); ++ const uint8_t * qs = x[ibl].qs; ++ const int8_t * q8 = y[ibl].qs; ++ int32_t sum = 0; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ const int ls1 = ((x[ibl].scales_l[ib] & 0xf) | ((h << 4) & 0x30)) - 32; ++ const int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; ++ h >>= 4; ++ const int8_t * values1 = iq4k_values + 16*(extra & 1); ++ const int8_t * values2 = iq4k_values + 8*(extra & 2); ++ extra >>= 2; ++ int sumi1 = 0, sumi2 = 0; ++ for (int j = 0; j < 16; ++j) { ++ sumi1 += q8[j+ 0] * values1[qs[j] & 0xf]; ++ sumi2 += q8[j+16] * values2[qs[j] >> 4]; ++ } ++ sum += ls1*sumi1 + ls2*sumi2; ++ qs += 16; ++ q8 += 32; ++ } ++ sumf += d4d8 * sum; ++ } ++ *s = sumf; ++ ++} ++ ++namespace { ++const int8_t iq4nl_index[241] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ++ 1, 17, 17, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ++ 3, 3, 3, 3, 3, 3, 19, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 20, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ++ 5, 5, 21, 21, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 22, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 23, 23, 8, 8, 8, 8, ++ 8, 8, 8, 8, 8, 8, 24, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 25, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 26, 26, ++ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 27, 27, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 28, 13, 13, 13, ++ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 29, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, ++ 14, 14, 14, 14, 30, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15 ++}; ++inline int best_index_iq4nl(const int8_t * values, float x) { ++ int ix = (int)x - values[0]; ++ if (ix < 0 || ix >= 241) return ix < 0 ? 0 : 15; ++ ix = iq4nl_index[ix]; ++ return ix < 16 ? ix : x - values[ix-16] < values[ix-15] - x ? ix-16 : ix-15; ++} ++ ++static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int block_size, const float * x, ++ block_iq4_k * y, ++ float * scales, float * weight, uint8_t * L, ++ const int8_t * values, ++ const float * quant_weights, ++ const int ntry) { ++ ++ GGML_ASSERT(super_block_size == 256 && block_size == 16); ++ ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j]; ++ sigma2 *= 2.f/super_block_size; ++ ++ memset(y, 0, sizeof(block_iq4_k)); ++ y->d = GGML_FP32_TO_FP16(0.f); ++ ++ uint16_t * scales_h = (uint16_t *)y->scales_h; ++ ++ const int8_t * shifted_values = values + 16; ++ ++ float max_scale = 0, amax_scale = 0; ++ uint16_t extra = 0; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = x + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ d = sumqx_p/sumq2_p; ++ bool is_shifted = false; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (is_shifted) extra |= (1 << ib); ++ scales[ib] = d; ++ float abs_d = fabsf(d); ++ if (abs_d > amax_scale) { ++ amax_scale = abs_d; max_scale = d; ++ } ++ } ++ float d = -max_scale/32; ++ y->d = GGML_FP32_TO_FP16(d); ++ y->extra = extra; ++ float id = d ? 1/d : 0.f; ++ float sumqx = 0, sumq2 = 0; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const int8_t * block_values = extra & (1 << ib) ? shifted_values : values; ++ int l = nearest_int(id*scales[ib]); ++ l = MAX(-32, MIN(31, l)); ++ float dl = d * l; ++ float idl = dl ? 1/dl : 0.f; ++ uint8_t * Lb = L + ib*block_size; ++ const float * xb = x + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ for (int j = 0; j < block_size; ++j) { ++ Lb[j] = best_index_iq4nl(block_values, idl*xb[j]); ++ float w = weight[j]; ++ float q = block_values[Lb[j]]*l; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ l += 32; ++ uint8_t l_l = l & 0xf; ++ uint8_t l_h = l >> 4; ++ if (ib%2 == 0) y->scales_l[ib/2] = l_l; ++ else y->scales_l[ib/2] |= (l_l << 4); ++ scales_h[ib/8] |= (l_h << 2*(ib%8)); ++ } ++ if (sumq2 > 0) y->d = GGML_FP32_TO_FP16(sumqx/sumq2); ++ ++ for (int i = 0; i < super_block_size/32; ++i) { ++ for (int j = 0; j < 16; ++j) { ++ y->qs[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4); ++ } ++ } ++} ++ ++} ++ ++void quantize_row_iq4_k_ref(const float * x, block_iq4_k * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq4_k(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_k(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq4_k * y = (block_iq4_k *)vy; ++ quantize_row_iq4_k_ref(x, y, k); ++} ++ ++size_t quantize_iq4_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ uint8_t L[QK_K]; ++ float weight[16]; ++ float scales[QK_K/16]; ++ auto q_func = [&L, &weight, &scales] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ block_iq4_k * iq4 = (block_iq4_k *)vy; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const float * qw = imatrix ? imatrix + QK_K*ibl : nullptr; ++ quantize_row_iq4_k_impl_bs16(QK_K, 16, x + QK_K*ibl, iq4 + ibl, ++ scales, weight, L, iq4k_values, qw, 7); ++ } ++ }; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_K, n_per_row); ++ QHelper helper(imatrix, user_data, n_per_row, 16); ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++// ++// ============================================== iq5_K ++// ++void dequantize_row_iq5_k(const block_iq5_k * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ const uint8_t * qs = x[i].qs; ++ const uint8_t * qh = x[i].qh; ++ const uint8_t * sl = x[i].scales_l; ++ const uint8_t * sh = x[i].scales_h; ++ ++ uint16_t extra = x[i].extra; ++ ++ int shift = 0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ ++ float dl1 = d * (((sl[2*ib64+0] & 0xf) | ((sh[ib64] << 4) & 0x30)) - 32); ++ float dl2 = d * (((sl[2*ib64+0] >> 4) | ((sh[ib64] << 2) & 0x30)) - 32); ++ float dl3 = d * (((sl[2*ib64+1] & 0xf) | ((sh[ib64] >> 0) & 0x30)) - 32); ++ float dl4 = d * (((sl[2*ib64+1] >> 4) | ((sh[ib64] >> 2) & 0x30)) - 32); ++ const int8_t * values1 = iq5nl_values + ((extra & 1) << 5); ++ const int8_t * values2 = iq5nl_values + ((extra & 2) << 4); ++ const int8_t * values3 = iq5nl_values + ((extra & 4) << 3); ++ const int8_t * values4 = iq5nl_values + ((extra & 8) << 2); ++ for (int j = 0; j < 16; ++j) { ++ y[j+ 0] = dl1 * values1[(qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4)]; ++ y[j+16] = dl2 * values2[(qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4)]; ++ y[j+32] = dl3 * values3[(qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3)]; ++ y[j+48] = dl4 * values4[(qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3)]; ++ } ++ y += 64; ++ qs += 32; ++ extra >>= 4; ++ shift += 2; ++ if (shift == 8) { qh += 32; shift = 0; } ++ } ++ ++ } ++} ++ ++void vec_dot_iq5_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ5_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ const int nb = n / QK_K; ++ ++ const block_iq5_k * x = (const block_iq5_k *)vx; ++ const block_q8_K * y = (const block_q8_K *)vy; ++ ++ float sumf = 0; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; ++ const uint8_t * qs = x[i].qs; ++ const uint8_t * qh = x[i].qh; ++ const uint8_t * sl = x[i].scales_l; ++ const uint8_t * sh = x[i].scales_h; ++ const int8_t * q8 = y[i].qs; ++ ++ uint16_t extra = x[i].extra; ++ ++ int shift = 0; ++ int sumb = 0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ ++ int dl1 = (((sl[2*ib64+0] & 0xf) | ((sh[ib64] << 4) & 0x30)) - 32); ++ int dl2 = (((sl[2*ib64+0] >> 4) | ((sh[ib64] << 2) & 0x30)) - 32); ++ int dl3 = (((sl[2*ib64+1] & 0xf) | ((sh[ib64] >> 0) & 0x30)) - 32); ++ int dl4 = (((sl[2*ib64+1] >> 4) | ((sh[ib64] >> 2) & 0x30)) - 32); ++ const int8_t * values1 = iq5nl_values + ((extra & 1) << 5); ++ const int8_t * values2 = iq5nl_values + ((extra & 2) << 4); ++ const int8_t * values3 = iq5nl_values + ((extra & 4) << 3); ++ const int8_t * values4 = iq5nl_values + ((extra & 8) << 2); ++ int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; ++ for (int j = 0; j < 16; ++j) { ++ sumi1 += q8[j+ 0] * values1[(qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4)]; ++ sumi2 += q8[j+16] * values2[(qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4)]; ++ sumi3 += q8[j+32] * values3[(qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3)]; ++ sumi4 += q8[j+48] * values4[(qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3)]; ++ } ++ sumb += dl1 * sumi1 + dl2 * sumi2 + dl3 * sumi3 + dl4 * sumi4; ++ q8 += 64; ++ qs += 32; ++ extra >>= 4; ++ shift += 2; ++ } ++ sumf += d * sumb; ++ ++ } ++ ++ *s = sumf; ++ ++} ++ ++namespace { ++const int8_t iq5nl_index[248] = { ++ 0, 0, 0, 0, 0, 0, 32, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 33, 33, 2, 2, 2, 2, 2, 2, 2, 2, 2, 34, 34, 3, 3, ++ 3, 3, 3, 3, 3, 3, 35, 35, 4, 4, 4, 4, 4, 4, 4, 36, 36, 5, 5, 5, 5, 5, 5, 5, 37, 37, 6, 6, 6, 6, 6, 6, ++ 6, 38, 7, 7, 7, 7, 7, 7, 39, 39, 8, 8, 8, 8, 8, 40, 40, 9, 9, 9, 9, 9, 41, 41, 10, 10, 10, 10, 10, 42, 11, 11, ++ 11, 11, 11, 43, 12, 12, 12, 12, 12, 44, 13, 13, 13, 13, 13, 45, 14, 14, 14, 14, 14, 46, 15, 15, 15, 15, 47, 47, 16, 16, 16, 16, ++ 48, 17, 17, 17, 17, 17, 49, 18, 18, 18, 18, 18, 50, 19, 19, 19, 19, 19, 51, 20, 20, 20, 20, 20, 52, 21, 21, 21, 21, 21, 53, 53, ++ 22, 22, 22, 22, 22, 54, 54, 23, 23, 23, 23, 23, 23, 55, 24, 24, 24, 24, 24, 24, 24, 56, 25, 25, 25, 25, 25, 25, 25, 57, 57, 26, ++ 26, 26, 26, 26, 26, 26, 58, 58, 27, 27, 27, 27, 27, 27, 27, 27, 59, 28, 28, 28, 28, 28, 28, 28, 28, 28, 60, 29, 29, 29, 29, 29, ++ 29, 29, 29, 29, 29, 61, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 62, 31, 31, 31, 31, 31, 31 ++}; ++inline int best_index_iq5nl(const int8_t * values, float x) { ++ int ix = (int)x - values[0]; ++ if (ix < 0 || ix >= 247) return ix < 0 ? 0 : 31; ++ ix = iq5nl_index[ix]; ++ return ix < 32 ? ix : x - values[ix-32] < values[ix-31] - x ? ix-32 : ix-31; ++} ++ ++void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ const int ntry = 5; ++ const float step = 1.f; ++ ++ block_iq5_k * y = (block_iq5_k *)vy; ++ ++ float scales[QK_K/16]; ++ float weight[16]; ++ ++ const int8_t * shifted_values = iq5nl_values + 32; ++ ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq5_k)); ++ y[ibl].d = GGML_FP32_TO_FP16(0.f); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 2*sumx2/QK_K; ++ ++ float max_scale = 0, max_abs_scale = 0; ++ uint16_t extra = 0; ++ ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < 16; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/iq5nl_values[0] : max/iq5nl_values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(iq5nl_values, al); ++ float q = iq5nl_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(iq5nl_values, -al); ++ q = iq5nl_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ d = sumqx_p/sumq2_p; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ bool is_shifted = false; ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry*step + iq5nl_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(iq5nl_values, al); ++ float q = iq5nl_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(iq5nl_values, -al); ++ q = iq5nl_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry*step + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (d) { ++ const int8_t * block_values = is_shifted ? shifted_values : iq5nl_values; ++ float sumqx = 0, sumq2 = 0; ++ id = 1/d; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(block_values, al); ++ float q = block_values[l]; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ if (sumq2 > 0) d = sumqx/sumq2; ++ } ++ scales[ib] = d; ++ if (is_shifted) extra |= (1 << ib); ++ ++ float abs_scale = fabsf(scales[ib]); ++ if (abs_scale > max_abs_scale) { ++ max_abs_scale = abs_scale; max_scale = scales[ib]; ++ } ++ ++ } ++ ++ if (!max_abs_scale) continue; ++ float d = -max_scale/32; ++ y[ibl].d = GGML_FP32_TO_FP16(d); ++ y[ibl].extra = extra; ++ ++ float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ int ls = nearest_int(id*scales[ib]); ++ ls = MAX(-32, MIN(31, ls)); ++ int uls = ls + 32; ++ y[ibl].scales_l[ib/2] |= ((uls & 0xf) << 4*(ib%2)); ++ y[ibl].scales_h[ib/4] |= ((uls >> 4) << 2*(ib%4)); ++ float dl = d * ls; ++ if (dl) { ++ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq5nl_values; ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float idl = 1/dl; ++ int ib32 = ib/2; ++ int offset = 16*(ib%2); ++ uint8_t * qs = y[ibl].qs + 32*(ib32/2) + offset; ++ uint8_t * qh = y[ibl].qh + 32*(ib32/8) + offset; ++ for (int j = 0; j < 16; ++j) { ++ const float al = idl*xb[j]; ++ int ibest = best_index_iq5nl(block_values, al); ++ qs[j] |= ((ibest & 0xf) << 4*(ib32%2)); ++ qh[j] |= ((ibest >> 4) << (ib32%8)); ++ float w = weight[j]; ++ float q = block_values[ibest]*ls; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2); ++ ++ } ++ ++} ++ ++} ++ ++void quantize_row_iq5_k_ref(const float * x, block_iq5_k * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq5_k(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq5_k(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq5_k * y = (block_iq5_k *)vy; ++ quantize_row_iq5_k_ref(x, y, k); ++} ++ ++size_t quantize_iq5_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ QHelper helper(imatrix, user_data, n_per_row, 16); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ5_K, n_per_row); ++ helper.quantize(nrows, src, dst, row_size, quantize_row_iq5_k_impl); ++ return nrows * row_size; ++} ++ ++// ++// ============================================== iq6_K ++// ++#define A_IQ6K -127.f ++#define B_IQ6K 6.2568f ++#define C_IQ6K 0.11218f ++#define D_IQ6K 0.0011972f ++#define S_IQ6K 1.f ++ ++void dequantize_row_iq6_k(const block_iq6_k * x, float * y, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ ++ for (int i = 0; i < nb; i++) { ++ ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ const uint8_t * qs = x[i].qs; ++ const uint8_t * qh = x[i].qh; ++ const int8_t * sl = x[i].scales; ++ ++ uint16_t extra = x[i].extra; ++ ++ int shift = 0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ ++ float dl1 = d * sl[4*ib64 + 0]; ++ float dl2 = d * sl[4*ib64 + 1]; ++ float dl3 = d * sl[4*ib64 + 2]; ++ float dl4 = d * sl[4*ib64 + 3]; ++ float m1 = extra & 1 ? S_IQ6K : 0; ++ float m2 = extra & 2 ? S_IQ6K : 0; ++ float m3 = extra & 4 ? S_IQ6K : 0; ++ float m4 = extra & 8 ? S_IQ6K : 0; ++ for (int j = 0; j < 16; ++j) { ++ float q1 = ((qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 0x03) << 4)); ++ float q2 = ((qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 0x03) << 4)); ++ float q3 = ((qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 0x0c) << 2)); ++ float q4 = ((qs[j+16] >> 4) | (((qh[j+16] >> shift) & 0x0c) << 2)); ++ y[j+ 0] = dl1 * (A_IQ6K + q1*(B_IQ6K + q1*(-C_IQ6K + q1*D_IQ6K)) + m1); ++ y[j+16] = dl2 * (A_IQ6K + q2*(B_IQ6K + q2*(-C_IQ6K + q2*D_IQ6K)) + m2); ++ y[j+32] = dl3 * (A_IQ6K + q3*(B_IQ6K + q3*(-C_IQ6K + q3*D_IQ6K)) + m3); ++ y[j+48] = dl4 * (A_IQ6K + q4*(B_IQ6K + q4*(-C_IQ6K + q4*D_IQ6K)) + m4); ++ } ++ y += 64; ++ qs += 32; ++ extra >>= 4; ++ shift += 4; ++ if (shift == 8) { qh += 32; shift = 0; } ++ } ++ ++ } ++} ++ ++void vec_dot_iq6_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ6_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++ GGML_ABORT("not implemented"); ++ ++ // TODO ++ //const int nb = n / QK_K; ++ ++ //const block_iq5_k * x = (const block_iq5_k *)vx; ++ //const block_q8_K * y = (const block_q8_K *)vy; ++ ++ //float sumf = 0; ++ ++ //for (int i = 0; i < nb; i++) { ++ ++ // const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; ++ // const uint8_t * qs = x[i].qs; ++ // const uint8_t * qh = x[i].qh; ++ // const uint8_t * sl = x[i].scales_l; ++ // const uint8_t * sh = x[i].scales_h; ++ // const int8_t * q8 = y[i].qs; ++ ++ // uint16_t extra = x[i].extra; ++ ++ // int shift = 0; ++ // int sumb = 0; ++ // for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ ++ // int dl1 = (((sl[2*ib64+0] & 0xf) | ((sh[ib64] << 4) & 0x30)) - 32); ++ // int dl2 = (((sl[2*ib64+0] >> 4) | ((sh[ib64] << 2) & 0x30)) - 32); ++ // int dl3 = (((sl[2*ib64+1] & 0xf) | ((sh[ib64] >> 0) & 0x30)) - 32); ++ // int dl4 = (((sl[2*ib64+1] >> 4) | ((sh[ib64] >> 2) & 0x30)) - 32); ++ // const int8_t * values1 = iq5nl_values + ((extra & 1) << 5); ++ // const int8_t * values2 = iq5nl_values + ((extra & 2) << 4); ++ // const int8_t * values3 = iq5nl_values + ((extra & 4) << 3); ++ // const int8_t * values4 = iq5nl_values + ((extra & 8) << 2); ++ // int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0; ++ // for (int j = 0; j < 16; ++j) { ++ // sumi1 += q8[j+ 0] * values1[(qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4)]; ++ // sumi2 += q8[j+16] * values2[(qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4)]; ++ // sumi3 += q8[j+32] * values3[(qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3)]; ++ // sumi4 += q8[j+48] * values4[(qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3)]; ++ // } ++ // sumb += dl1 * sumi1 + dl2 * sumi2 + dl3 * sumi3 + dl4 * sumi4; ++ // q8 += 64; ++ // qs += 32; ++ // extra >>= 4; ++ // shift += 2; ++ // } ++ // sumf += d * sumb; ++ ++ //} ++ ++ //*s = sumf; ++ ++} ++ ++namespace { ++ ++inline int best_index(int n, const float * val, float x) { ++ if (x <= val[0]) return 0; ++ if (x >= val[n-1]) return n-1; ++ int ml = 0, mu = n-1; ++ while (mu-ml > 1) { ++ int mav = (ml+mu)/2; ++ if (x < val[mav]) mu = mav; else ml = mav; ++ } ++ return x - val[mu-1] < val[mu] - x ? mu-1 : mu; ++} ++uint8_t iq6nl_index[249] = { ++ 0, 0, 0, 64, 1, 1, 1, 1, 1, 65, 2, 2, 2, 2, 2, 66, 3, 3, 3, 3, 67, 67, 4, 4, 4, 4, 68, 5, 5, 5, 5, 69, ++ 69, 6, 6, 6, 70, 70, 7, 7, 7, 71, 8, 8, 8, 72, 72, 9, 9, 9, 73, 73, 10, 10, 10, 74, 11, 11, 11, 75, 12, 12, 12, 76, ++ 13, 13, 13, 77, 14, 14, 14, 78, 15, 15, 79, 79, 16, 16, 80, 17, 17, 81, 81, 18, 18, 82, 19, 19, 83, 83, 20, 84, 84, 21, 85, 85, ++ 22, 86, 86, 23, 87, 87, 24, 88, 88, 25, 89, 89, 26, 90, 90, 27, 91, 91, 28, 92, 29, 93, 93, 30, 94, 94, 31, 95, 95, 32, 96, 33, ++ 97, 97, 34, 98, 98, 35, 99, 99, 36, 100, 100, 37, 101, 38, 102, 102, 39, 103, 103, 40, 104, 104, 41, 41, 105, 42, 42, 106, 106, 43, 107, 107, ++ 44, 108, 108, 45, 45, 109, 46, 46, 46, 110, 47, 47, 111, 111, 48, 48, 112, 49, 49, 49, 113, 50, 50, 50, 114, 51, 51, 51, 115, 52, 52, 52, ++ 116, 116, 53, 53, 53, 117, 54, 54, 54, 118, 118, 55, 55, 55, 119, 119, 56, 56, 56, 120, 120, 57, 57, 57, 121, 121, 58, 58, 58, 58, 122, 59, ++ 59, 59, 59, 123, 123, 60, 60, 60, 60, 124, 61, 61, 61, 61, 61, 125, 62, 62, 62, 62, 62, 126, 63, 63, 63, ++}; ++inline int best_index_iq6nl(const float * values, float x) { ++ int ix = (int)(x - values[0]); ++ if (ix < 0 || ix >= 249) return ix < 0 ? 0 : 63; ++ ix = iq6nl_index[ix]; ++ return ix < 64 ? ix : x - values[ix-64] < values[ix-63] - x ? ix-64 : ix-63; ++ //if (x <= val[0]) return 0; ++ //if (x >= val[63]) return 63; ++ //int index = iq6nl_index[int(x - val[0])]; ++ //return index < 64 ? index : x - val[index-64] < val[index-63] - x ? index - 64 : index - 63; ++} ++ ++ ++void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, const float * values, const float * shifted_values) { ++ const int ntry = 5; ++ const float step = 1.f; ++ ++ block_iq6_k * y = (block_iq6_k *)vy; ++ ++ float scales[QK_K/16]; ++ float weight[16]; ++ ++ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq6_k)); ++ y[ibl].d = GGML_FP32_TO_FP16(0.f); ++ ++ const float * xbl = x + ibl*QK_K; ++ float sumx2 = 0; ++ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j]; ++ const float sigma2 = 2*sumx2/QK_K; ++ ++ float max_scale = 0, max_abs_scale = 0; ++ uint16_t extra = 0; ++ ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < 16; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ //int l = best_index(64, values, al); ++ int l = best_index_iq6nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ //l = best_index(64, values, -al); ++ l = best_index_iq6nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ d = sumqx_p/sumq2_p; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ bool is_shifted = false; ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry*step + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ //int l = best_index(64, values, al); ++ int l = best_index_iq6nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ //l = best_index(64, values, -al); ++ l = best_index_iq6nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry*step + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ //int l = best_index(64, shifted_values, al); ++ int l = best_index_iq6nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ //l = best_index(64, shifted_values, -al); ++ l = best_index_iq6nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (d) { ++ const float * block_values = is_shifted ? shifted_values : values; ++ float sumqx = 0, sumq2 = 0; ++ id = 1/d; ++ for (int j = 0; j < 16; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ //int l = best_index(64, block_values, al); ++ int l = best_index_iq6nl(block_values, al); ++ float q = block_values[l]; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ if (sumq2 > 0) d = sumqx/sumq2; ++ } ++ scales[ib] = d; ++ if (is_shifted) extra |= (1 << ib); ++ ++ float abs_scale = fabsf(scales[ib]); ++ if (abs_scale > max_abs_scale) { ++ max_abs_scale = abs_scale; max_scale = scales[ib]; ++ } ++ ++ } ++ ++ if (!max_abs_scale) continue; ++ float d = -max_scale/127; ++ y[ibl].d = GGML_FP32_TO_FP16(d); ++ y[ibl].extra = extra; ++ ++ float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ int ls = nearest_int(id*scales[ib]); ++ ls = MAX(-127, MIN(127, ls)); ++ y[ibl].scales[ib] |= ls; ++ float dl = d * ls; ++ if (dl) { ++ const float * block_values = y[ibl].extra & (1 << ib) ? shifted_values : values; ++ const float * xb = xbl + 16*ib; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*QK_K + ib*16; ++ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j]; ++ } ++ float idl = 1/dl; ++ int ib32 = ib/2; ++ int offset = 16*(ib%2); ++ uint8_t * qs = y[ibl].qs + 32*(ib32/2) + offset; ++ uint8_t * qh = y[ibl].qh + 32*(ib32/4) + offset; ++ for (int j = 0; j < 16; ++j) { ++ const float al = idl*xb[j]; ++ //int ibest = best_index(64, block_values, al); ++ int ibest = best_index_iq6nl(block_values, al); ++ qs[j] |= ((ibest & 0xf) << 4*(ib32%2)); ++ qh[j] |= ((ibest >> 4) << 2*(ib32%4)); ++ float w = weight[j]; ++ float q = block_values[ibest]*ls; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2); ++ ++ } ++} ++ ++} ++ ++void quantize_row_iq6_k_ref(const float * x, block_iq6_k * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq6_k(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq6_k(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq6_k * y = (block_iq6_k *)vy; ++ quantize_row_iq6_k_ref(x, y, k); ++} ++ ++size_t quantize_iq6_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ float values[128]; ++ for (int i = 0; i < 64; ++i) { ++ values[i] = iq6nl_values[i]; ++ values[i+64] = values[i] + S_IQ6K; ++ } ++ auto q_func = [values] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq6_k_impl(x, vy, n_per_row, imatrix, values, values + 64); ++ }; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ6_K, n_per_row); ++ QHelper helper(imatrix, user_data, n_per_row, 16); ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++namespace { ++template ++void iqk_quantize_row_q8_K_T(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ const int nb = k / QK_K; ++ iqk_block_q8_K * y = (iqk_block_q8_K *)vy; // opencoti F5-opt W2 (#290): ik layout has extra `sum` ++#ifdef __AVX2__ ++ const __m256 signBit = _mm256_set1_ps(-0.0f); ++ const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ for (int i = 0; i < nb; i++) { ++ const float * xb = x + i*QK_K; ++ __m256 maxAbs = _mm256_setzero_ps(); ++ const float * xx = xb; ++ for (int ib = 0; ib < QK_K/8; ++ib) { ++ const __m256 v = _mm256_loadu_ps(xx); xx += 8; ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps(signBit, v)); ++ } ++ const float maxScalar = hmax_f32_8(maxAbs); ++ const float d = maxScalar / 127.f; ++ y[i].d = d; ++ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; ++ const __m256 mul = _mm256_set1_ps( id ); ++ xx = xb; ++ int8_t * q8 = y[i].qs; ++ int block_sum_i32 = 0; ++ float block_sum_f32 = 0; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ __m256 v0 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v1 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v2 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v3 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST); ++ v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST); ++ v2 = _mm256_round_ps(v2, _MM_ROUND_NEAREST); ++ v3 = _mm256_round_ps(v3, _MM_ROUND_NEAREST); ++ __m256i i0 = _mm256_cvtps_epi32(v0); ++ __m256i i1 = _mm256_cvtps_epi32(v1); ++ __m256i i2 = _mm256_cvtps_epi32(v2); ++ __m256i i3 = _mm256_cvtps_epi32(v3); ++ if constexpr (q8_type == 1) { ++ int bsum = hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))); ++ auto bs = (float *)y[i].bsums; ++ bs[ib] = d*bsum; ++ block_sum_f32 += bs[ib]; ++ } else { ++ y[i].bsums[2*ib+0] = hsum_i32_8(_mm256_add_epi32(i0, i1)); ++ y[i].bsums[2*ib+1] = hsum_i32_8(_mm256_add_epi32(i2, i3)); ++ block_sum_i32 += y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]; ++ } ++ i0 = _mm256_packs_epi32( i0, i1 ); ++ i2 = _mm256_packs_epi32( i2, i3 ); ++ i0 = _mm256_packs_epi16( i0, i2 ); ++ i0 = _mm256_permutevar8x32_epi32( i0, perm ); ++ _mm256_storeu_si256((__m256i *)q8, i0); ++ q8 += 32; ++ } ++ if constexpr (q8_type == 1) { ++ y[i].sum = block_sum_f32; ++ } else { ++ y[i].sum = d*block_sum_i32; ++ } ++ //if constexpr (q8_type == 2) { ++ // auto bs = (float *)y[i].bsums; ++ // float sum = 0; ++ // for (int ib = 0; ib < QK_K/32; ++ib) sum += bs[ib]; ++ // bs[0] = sum; ++ //} ++ } ++#else ++ for (int i = 0; i < nb; i++) { ++ ++ float max = 0; ++ float amax = 0; ++ for (int j = 0; j < QK_K; ++j) { ++ float ax = fabsf(x[j]); ++ if (ax > amax) { ++ amax = ax; max = x[j]; ++ } ++ } ++ if (!amax) { ++ y[i].d = 0; ++ memset(y[i].qs, 0, QK_K); ++ x += QK_K; ++ continue; ++ } ++ //const float iscale = -128.f/max; ++ // We need this change for IQ2_XXS, else the AVX implementation becomes very awkward ++ const float iscale = -127.f/max; ++ for (int j = 0; j < QK_K; ++j) { ++ int v = nearest_int(iscale*x[j]); ++ y[i].qs[j] = MIN(127, v); ++ } ++ float d = 1/iscale; ++ if constexpr (q8_type == 1) { ++ auto bs = (float *)y[i].bsums; ++ float sum = 0; ++ for (int j = 0; j < QK_K/32; ++j) { ++ int sum = 0; ++ for (int ii = 0; ii < 32; ++ii) { ++ sum += y[i].qs[j*32 + ii]; ++ } ++ bs[j] = d*sum; ++ sum += bs[j]; ++ } ++ y[i].sum = sum; ++ } else { ++ int tot = 0; ++ for (int j = 0; j < QK_K/16; ++j) { ++ int sum = 0; ++ for (int ii = 0; ii < 16; ++ii) { ++ sum += y[i].qs[j*16 + ii]; ++ } ++ y[i].bsums[j] = sum; ++ tot += sum; ++ } ++ y[i].sum = d*tot; ++ } ++ y[i].d = d; ++ x += QK_K; ++ } ++#endif ++} ++} ++ ++void iqk_quantize_row_q8_K(const float * x, void * vy, int64_t k) { ++ iqk_quantize_row_q8_K_T<0>(x, vy, k); ++} ++ ++void quantize_row_q8_K32(const float * x, void * vy, int64_t k) { ++ iqk_quantize_row_q8_K_T<1>(x, vy, k); ++} ++ ++void quantize_row_q8_KR8(const float * x, void * vy, int64_t k) { ++ iqk_quantize_row_q8_K_T<2>(x, vy, k); ++} ++ ++namespace { ++// TODO: merge this with the above template ++void iqk_quantize_row_q8_K128(const float * x, void * vy, int64_t k) { ++ constexpr int kBlockSize = 128; ++ assert(k % kBlockSize == 0); ++ const int nb = k / kBlockSize; ++ auto y = (block_q8_K128 *)vy; ++#ifdef __AVX2__ ++ const __m256 signBit = _mm256_set1_ps(-0.0f); ++ const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ for (int i = 0; i < nb; i++) { ++ const float * xb = x + i*kBlockSize; ++ __m256 maxAbs = _mm256_setzero_ps(); ++ const float * xx = xb; ++ for (int ib = 0; ib < kBlockSize/8; ++ib) { ++ const __m256 v = _mm256_loadu_ps(xx); xx += 8; ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps(signBit, v)); ++ } ++ const float maxScalar = hmax_f32_8(maxAbs); ++ const float d = maxScalar / 127.f; ++ y[i].d = d; ++ const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; ++ const __m256 mul = _mm256_set1_ps( id ); ++ xx = xb; ++ int8_t * q8 = y[i].qs; ++ for (int ib = 0; ib < kBlockSize/32; ++ib) { ++ __m256 v0 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v1 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v2 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ __m256 v3 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8; ++ v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST); ++ v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST); ++ v2 = _mm256_round_ps(v2, _MM_ROUND_NEAREST); ++ v3 = _mm256_round_ps(v3, _MM_ROUND_NEAREST); ++ __m256i i0 = _mm256_cvtps_epi32(v0); ++ __m256i i1 = _mm256_cvtps_epi32(v1); ++ __m256i i2 = _mm256_cvtps_epi32(v2); ++ __m256i i3 = _mm256_cvtps_epi32(v3); ++ y[i].bsums[ib] = hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))); ++ i0 = _mm256_packs_epi32( i0, i1 ); ++ i2 = _mm256_packs_epi32( i2, i3 ); ++ i0 = _mm256_packs_epi16( i0, i2 ); ++ i0 = _mm256_permutevar8x32_epi32( i0, perm ); ++ _mm256_storeu_si256((__m256i *)q8, i0); ++ q8 += 32; ++ } ++ } ++#elif defined __ARM_NEON ++ int32x4_t ival[8]; ++ for (int i = 0; i < nb; i++) { ++ const float * xb = x + i*kBlockSize; ++ auto vmax = vdupq_n_f32(0.f); ++ for (int j = 0; j < kBlockSize; j += 4) { ++ vmax = vmaxq_f32(vmax, vabsq_f32(vld1q_f32(xb + j))); ++ } ++ auto smax = vmaxvq_f32(vmax); ++ if (!smax) { ++ std::memset(&y[i], 0, sizeof(y[i])); ++ continue; ++ } ++ y[i].d = smax/127; ++ auto vid = vdupq_n_f32(127/smax); ++ for (int ib = 0; ib < kBlockSize/32; ++ib) { ++ auto isum = vdupq_n_s32(0); ++ for (int k = 0; k < 8; ++k) { ++ auto val = vld1q_f32(xb + 32*ib + 4*k); ++ ival[k] = vcvtnq_s32_f32(vmulq_f32(val, vid)); ++ isum = vaddq_s32(isum, ival[k]); ++ } ++ y[i].bsums[ib] = vaddvq_s32(isum); ++ for (int k = 0; k < 4; ++k) { ++ auto i16 = vcombine_s16(vmovn_s32(ival[2*k+0]), vmovn_s32(ival[2*k+1])); ++ vst1_s8(y[i].qs + 32*ib + 8*k, vmovn_s16(i16)); ++ } ++ } ++ } ++#else ++ for (int i = 0; i < nb; i++) { ++ ++ float amax = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ float ax = std::abs(x[j]); ++ amax = std::max(amax, ax); ++ } ++ if (!amax) { ++ y[i].d = 0; ++ memset(y[i].qs, 0, kBlockSize); ++ memset(y[i].bsums, 0, kBlockSize/32*(sizeof(int16_t))); ++ x += kBlockSize; ++ continue; ++ } ++ const float iscale = 127.f/amax; ++ for (int j = 0; j < kBlockSize; ++j) { ++ int v = nearest_int(iscale*x[j]); ++ y[i].qs[j] = v; ++ } ++ for (int j = 0; j < kBlockSize/32; ++j) { ++ int sum = 0; ++ for (int ii = 0; ii < 32; ++ii) { ++ sum += y[i].qs[j*32 + ii]; ++ } ++ y[i].bsums[j] = sum; ++ } ++ y[i].d = 1/iscale; ++ x += kBlockSize; ++ } ++#endif ++} ++// TODO: merge this with the above template ++void iqk_quantize_row_q8_KV(const float * x, void * vy, int64_t k) { ++ assert(k % 32 == 0); ++ auto dptr = (float *)vy; ++ auto q8 = (int8_t *)(dptr + 2); ++#ifdef __AVX2__ ++ const __m256 signBit = _mm256_set1_ps(-0.0f); ++ const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); ++ __m256 maxAbs = _mm256_setzero_ps(); ++ for (int ib = 0; ib < k/8; ++ib) { ++ const __m256 v = _mm256_loadu_ps(x + 8*ib); ++ maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps(signBit, v)); ++ } ++ const float maxScalar = hmax_f32_8(maxAbs); ++ if (!maxScalar) { ++ dptr[0] = dptr[1] = 0; ++ std::memset(q8, 0, k*sizeof(int8_t)); ++ return; ++ } ++ dptr[0] = maxScalar / 127.f; ++ auto mul = _mm256_set1_ps(1/dptr[0]); ++ auto isum = _mm256_setzero_si256(); ++ for (int i = 0; i < k/32; i++) { ++ __m256 v0 = _mm256_mul_ps(mul, _mm256_loadu_ps(x + 32*i + 0)); ++ __m256 v1 = _mm256_mul_ps(mul, _mm256_loadu_ps(x + 32*i + 8)); ++ __m256 v2 = _mm256_mul_ps(mul, _mm256_loadu_ps(x + 32*i + 16)); ++ __m256 v3 = _mm256_mul_ps(mul, _mm256_loadu_ps(x + 32*i + 24)); ++ v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST); ++ v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST); ++ v2 = _mm256_round_ps(v2, _MM_ROUND_NEAREST); ++ v3 = _mm256_round_ps(v3, _MM_ROUND_NEAREST); ++ __m256i i0 = _mm256_cvtps_epi32(v0); ++ __m256i i1 = _mm256_cvtps_epi32(v1); ++ __m256i i2 = _mm256_cvtps_epi32(v2); ++ __m256i i3 = _mm256_cvtps_epi32(v3); ++ isum = _mm256_add_epi32(isum, _mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))); ++ i0 = _mm256_packs_epi32( i0, i1 ); ++ i2 = _mm256_packs_epi32( i2, i3 ); ++ i0 = _mm256_packs_epi16( i0, i2 ); ++ i0 = _mm256_permutevar8x32_epi32( i0, perm ); ++ _mm256_storeu_si256((__m256i *)q8, i0); ++ q8 += 32; ++ } ++ auto iptr = (int32_t *)(dptr + 1); ++ iptr[0] = hsum_i32_8(isum); ++#elif defined __ARM_NEON ++ int32x4_t ival[8]; ++ auto vmax = vdupq_n_f32(0.f); ++ for (int j = 0; j < k; j += 4) { ++ vmax = vmaxq_f32(vmax, vabsq_f32(vld1q_f32(x + j))); ++ } ++ auto smax = vmaxvq_f32(vmax); ++ if (!smax) { ++ dptr[0] = dptr[1] = 0; ++ std::memset(q8, 0, k*sizeof(int8_t)); ++ return; ++ } ++ dptr[0] = smax/127; ++ auto vid = vdupq_n_f32(1/dptr[0]); ++ auto isum = vdupq_n_s32(0); ++ for (int ib = 0; ib < k/32; ++ib) { ++ auto xb = x + 32*ib; ++ for (int k = 0; k < 8; ++k) { ++ auto val = vld1q_f32(xb + 4*k); ++ ival[k] = vcvtnq_s32_f32(vmulq_f32(val, vid)); ++ isum = vaddq_s32(isum, ival[k]); ++ } ++ for (int k = 0; k < 4; ++k) { ++ auto i16 = vcombine_s16(vmovn_s32(ival[2*k+0]), vmovn_s32(ival[2*k+1])); ++ vst1_s8(q8, vmovn_s16(i16)); ++ q8 += 8; ++ } ++ } ++ auto iptr = (int32_t *)(dptr + 1); ++ iptr[0] = vaddvq_s32(isum); ++#else ++ float amax = 0; ++ for (int j = 0; j < k; ++j) { ++ float ax = std::abs(x[j]); ++ amax = std::max(amax, ax); ++ } ++ if (!amax) { ++ dptr[0] = dptr[1] = 0; ++ std::memset(q8, 0, k*sizeof(int8_t)); ++ return; ++ } ++ dptr[0] = amax/127; ++ float id = 1/dptr[0]; ++ int isum = 0; ++ for (int i = 0; i < k; i++) { ++ q8[i] = nearest_int(id*x[i]); ++ isum += q8[i]; ++ } ++ auto iptr = (int32_t *)(dptr + 1); ++ iptr[0] = isum; ++#endif ++} ++} ++ ++void quantize_row_q8_K128(const float * x, void * vy, int64_t k) { ++ iqk_quantize_row_q8_K128(x, vy, k); ++} ++ ++// ============================== MXFP4 ++ ++namespace { ++inline int best_index_mxfp4(float d, const int8_t * values, float x) { ++ float best = std::abs(x - d*values[0]); ++ int index = 0; ++ for (int j = 1; j < 16; ++j) { ++ float diff = std::abs(x - d*values[j]); ++ if (diff < best) { best = diff; index = j; } ++ } ++ return index; ++} ++static void quantize_row_mxfp4_impl(int n_per_row, const float * x, char * cy, ++ [[maybe_unused]] float * weight, ++ const int8_t * values, ++ [[maybe_unused]] const float * quant_weights, ++ [[maybe_unused]] const int ntry) { ++ ++ GGML_ASSERT(n_per_row % QK_MXFP4 == 0); ++ GGML_UNUSED(quant_weights); ++ ++ block_mxfp4 * y = (block_mxfp4 *)cy; ++ ++ //int last_ibl = -1; ++ //float sigma2 = 0; ++ ++ //const uint8_t e = (uint8_t) (floorf(log2f(amax)) - 2 + 127); ++ // -> log2f(amax) ~ e - 125 -> amax = 2^(e - 125) ++ //const float d = GGML_E8M0_TO_FP32_HALF(e); ++ ++ for (int ib = 0; ib < n_per_row/QK_MXFP4; ++ib) { ++ memset(&y[ib], 0, sizeof(block_mxfp4)); ++ const float * xb = x + ib*QK_MXFP4; ++ //if (int ibl = ib/(QK_K/QK_MXFP4); ibl != last_ibl) { ++ // int n = std::min(QK_K, n_per_row - ib*QK_MXFP4); ++ // float sumx2 = 0; ++ // for (int j = 0; j < n; ++j) sumx2 += xb[j]*xb[j]; ++ // sigma2 = 2.0f*sumx2/n; ++ // last_ibl = ibl; ++ //} ++ //if (quant_weights) { ++ // const float * qw = quant_weights + ib*QK_MXFP4; ++ // for (int j = 0; j < QK_MXFP4; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ //} else { ++ // for (int j = 0; j < QK_MXFP4; ++j) weight[j] = xb[j]*xb[j]; ++ //} ++ float amax = 0; ++ for (int j = 0; j < QK_MXFP4; ++j) { ++ float ax = fabsf(xb[j]); ++ amax = std::max(amax, ax); ++ } ++ if (!amax) { ++ continue; ++ } ++ const uint8_t e = (uint8_t) (floorf(log2f(amax)) - 2 + 127); ++ const float d = GGML_E8M0_TO_FP32_HALF(e); ++ y[ib].e = e; ++ for (int j = 0; j < QK_MXFP4/2; ++j) { ++ uint8_t v0 = best_index_mxfp4(d, values, xb[j]); ++ uint8_t v1 = best_index_mxfp4(d, values, xb[j+QK_MXFP4/2]); ++ y[ib].qs[j] = v0 | (v1 << 4); ++ } ++ } ++} ++} ++ ++void iqk_quantize_row_mxfp4_ref(const float * x, block_mxfp4 * y, int64_t k) { ++ iqk_quantize_mxfp4(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void iqk_quantize_row_mxfp4(const float * x, void * y, int64_t k) { ++ iqk_quantize_mxfp4(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++size_t iqk_quantize_mxfp4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ constexpr int kBlockSize = QK_MXFP4; ++ GGML_ASSERT(n_per_row%kBlockSize == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_MXFP4, n_per_row); ++ char * qrow = (char *)dst; ++ float weight[kBlockSize]; ++ for (int64_t row = 0; row < nrows; ++row) { ++ quantize_row_mxfp4_impl(n_per_row, src, qrow, weight, kvalues_mxfp4, imatrix, 7); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrows * row_size; ++} ++ ++void iqk_dequantize_row_mxfp4(const block_mxfp4 * x, float * y, int64_t k) { ++ constexpr int kBlockSize = QK_MXFP4; ++ GGML_ASSERT(k%kBlockSize == 0); ++ int nblock = k/kBlockSize; ++ for (int ib = 0; ib < nblock; ++ib) { ++ float d = GGML_E8M0_TO_FP32_HALF(x[ib].e); ++ for (int j = 0; j < kBlockSize/2; ++j) { ++ y[j ] = d * kvalues_mxfp4[x[ib].qs[j] & 0xf]; ++ y[j+kBlockSize/2] = d * kvalues_mxfp4[x[ib].qs[j] >> 4]; ++ } ++ y += kBlockSize; ++ } ++} ++ ++void vec_dot_mxfp4_q8_0_x4(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_MXFP4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK_MXFP4 == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ //const block_mxfp4 * x = (const block_mxfp4 *)vx; ++ //const block_q8_K * y = (const block_q8_K *)vy; ++ //int nblock = n/QK_MXFP4; ++ //float sumf = 0; ++ //for (int ibl = 0; ibl < nblock; ++ibl) { ++ // //int sumi = 0; ++ // auto qy = y[ibl].qs; ++ // auto qx = x[ibl].qs; ++ // float db = d * y[ibl].d; ++ // for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ // float dl = db * ((x[ibl].scales[ib] & 254) - 127); ++ // //int ls = (x[ibl].scales[ib] & 254) - 127; ++ // const int8_t * values = iq4k_values + ((x[ibl].scales[ib] & 1) << 4); ++ // int suml = 0; ++ // for (int j = 0; j < kBlockSize/2; ++j) { ++ // suml += qy[j ] * values[qx[j] & 0xf] ++ // + qy[j + kBlockSize/2] * values[qx[j] >> 4]; ++ // } ++ // sumf += dl * suml; ++ // //sumi += ls * suml; ++ // qy += kBlockSize; ++ // qx += kBlockSize/2; ++ // } ++ // //sumf += d * y[ibl].d * sumi; ++ //} ++ //*s = sumf; ++} ++ ++namespace { ++static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int block_size, ++ int n_per_row, const float * x, char * cy, ++ float * all_scales, float * weight, ++ const int8_t * values, ++ const float * quant_weights, ++ const int ntry) { ++ ++ //GGML_ASSERT(super_block_size == 256 && block_size == 128); ++ ++ float * dptr = (float *)cy; ++ block_iq4_ks * y = (block_iq4_ks *)(dptr + 1); ++ ++ const int8_t * shifted_values = values + 16; ++ ++ float amax_scale = 0; ++ ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ memset(&y[ibl], 0, sizeof(block_iq4_ks)); ++ const float * xbl = x + ibl*super_block_size; ++ auto scales = all_scales + ibl*(super_block_size/block_size); ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ d = sumqx_p/sumq2_p; ++ bool is_shifted = false; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq4nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq4nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (is_shifted) y[ibl].scales[ib] = 0x01; ++ scales[ib] = d; ++ amax_scale = std::max(amax_scale, std::abs(d)); ++ } ++ } ++ float d = amax_scale/127; ++ *dptr = d; ++ if (!d) return; ++ float id = d ? 1/d : 0.f; ++ float sumqx = 0, sumq2 = 0; ++ //float mse = 0; ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ const float * xbl = x + ibl*super_block_size; ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ auto scales = all_scales + (super_block_size/block_size)*ibl; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const int8_t * block_values = y[ibl].scales[ib] & 0x01 ? shifted_values : values; ++ int l = nearest_int(0.5f*(id*scales[ib]+127.f)); ++ l = std::max(0, std::min(127, l)) << 1; ++ //printf("d = %g, id = %g, scales = %g, l = %d, dl = %g\n", d, id, scales[ib], l, d*(l - 127)); ++ y[ibl].scales[ib] |= l; ++ l -= 127; ++ float dl = d * l; ++ float idl = dl ? 1/dl : 0.f; ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ auto qs = y[ibl].qs + ib*(block_size/2); ++ for (int j = 0; j < block_size/2; ++j) { ++ uint8_t i1 = best_index_iq4nl(block_values, idl*xb[j]); ++ uint8_t i2 = best_index_iq4nl(block_values, idl*xb[j+block_size/2]); ++ qs[j] = i1 | (i2 << 4); ++ float w1 = weight[j]; ++ float w2 = weight[j+block_size/2]; ++ float q1 = block_values[i1]*l; ++ float q2 = block_values[i2]*l; ++ sumqx += w1*q1*xb[j] + w2*q2*xb[j+block_size/2]; ++ sumq2 += w1*q1*q1 + w2*q2*q2; ++ //float diff = xb[j] - d*q1; mse += diff*diff; ++ //diff = xb[j+block_size/2] - d*q2; mse += diff*diff; ++ } ++ } ++ } ++ //printf("rmse = %g\n", sqrt(mse/n_per_row)); ++ if (sumq2 > 0) *dptr = sumqx/sumq2; ++} ++} ++ ++void quantize_row_iq4_ks_ref(const float * x, block_iq4_ks * y, int64_t k) { ++ quantize_iq4_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_ks(const float * x, void * y, int64_t k) { ++ quantize_iq4_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++size_t quantize_iq4_ks(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_KS, n_per_row); ++ float weight[kBlockSize]; ++ std::vector all_scales(n_per_row/kBlockSize); ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ auto q_func = [&all_scales, &weight, block_size = kBlockSize] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq4_k_impl_bs128(QK_K, block_size, n_per_row, x, (char *)vy, all_scales.data(), weight, iq4k_values, imatrix, 7); ++ }; ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq4_ks(const block_iq4_ks * x, float * y, int64_t k) { ++ constexpr int kBlockSize = 32; //128; ++ GGML_ASSERT(k%QK_K == 0); ++ const float * dptr = (const float *)x; ++ float d = *dptr; ++ x = (const block_iq4_ks *)(dptr + 1); ++ int nblock = k/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto qs = x[ibl].qs; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ float dl = d * ((int)(x[ibl].scales[ib] & 254) - 127); ++ const int8_t * values = iq4k_values + ((x[ibl].scales[ib] & 1) << 4); ++ for (int j = 0; j < kBlockSize/2; ++j) { ++ y[j ] = dl * values[qs[j] & 0xf]; ++ y[j+kBlockSize/2] = dl * values[qs[j] >> 4]; ++ } ++ y += kBlockSize; ++ qs += kBlockSize/2; ++ } ++ } ++} ++ ++void vec_dot_iq4_ks_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ constexpr int kBlockSize = 32; ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_KS, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ const float * dptr = (const float *)vx; ++ const float d = *dptr; ++ //printf("%s: n = %d, d = %g\n", __func__, n, d); ++ const block_iq4_ks * x = (const block_iq4_ks *)(dptr + 1); ++ const block_q8_K * y = (const block_q8_K *)vy; ++ int nblock = n/QK_K; ++ float sumf = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ //int sumi = 0; ++ auto qy = y[ibl].qs; ++ auto qx = x[ibl].qs; ++ float db = d * y[ibl].d; ++ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) { ++ float dl = db * ((x[ibl].scales[ib] & 254) - 127); ++ //int ls = (x[ibl].scales[ib] & 254) - 127; ++ const int8_t * values = iq4k_values + ((x[ibl].scales[ib] & 1) << 4); ++ int suml = 0; ++ for (int j = 0; j < kBlockSize/2; ++j) { ++ suml += qy[j ] * values[qx[j] & 0xf] ++ + qy[j + kBlockSize/2] * values[qx[j] >> 4]; ++ } ++ sumf += dl * suml; ++ //sumi += ls * suml; ++ qy += kBlockSize; ++ qx += kBlockSize/2; ++ } ++ //sumf += d * y[ibl].d * sumi; ++ } ++ *s = sumf; ++} ++ ++namespace { ++static void quantize_row_iq5_ks_impl(const int super_block_size, const int block_size, ++ int n_per_row, const float * x, char * cy, ++ float * all_scales, float * weight, ++ const int8_t * values, ++ const float * quant_weights, ++ const int ntry) { ++ ++ float * dptr = (float *)cy; ++ dptr[0] = 0; ++ block_iq5_ks * y = (block_iq5_ks *)(dptr + 1); ++ ++ const int8_t * shifted_values = values + 32; ++ ++ float amax_scale = 0; ++ ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ memset(&y[ibl], 0, sizeof(block_iq5_ks)); ++ const float * xbl = x + ibl*super_block_size; ++ auto scales = all_scales + ibl*(super_block_size/block_size); ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float d = ntry > 0 ? -max/values[0] : max/values[0]; ++ float id = 1/d; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ d = sumqx_p/sumq2_p; ++ bool is_shifted = false; ++ float best = d*sumqx_p; ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d*sumqx_m; ++ } ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ id = (itry + values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(values, al); ++ float q = values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(values, -al); ++ q = values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false; ++ } ++ id = (itry + shifted_values[0])/max; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float w = weight[j]; ++ float al = id*xb[j]; ++ int l = best_index_iq5nl(shifted_values, al); ++ float q = shifted_values[l]; ++ sumqx_p += w*q*xb[j]; ++ sumq2_p += w*q*q; ++ l = best_index_iq5nl(shifted_values, -al); ++ q = shifted_values[l]; ++ sumqx_m += w*q*xb[j]; ++ sumq2_m += w*q*q; ++ } ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true; ++ } ++ } ++ if (is_shifted) y[ibl].scales[ib] = 0x01; ++ scales[ib] = d; ++ amax_scale = std::max(amax_scale, std::abs(d)); ++ } ++ } ++ float d = amax_scale/127; ++ *dptr = d; ++ if (!d) return; ++ float id = d ? 1/d : 0.f; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ const float * xbl = x + ibl*super_block_size; ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ auto scales = all_scales + (super_block_size/block_size)*ibl; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const int8_t * block_values = y[ibl].scales[ib] & 0x01 ? shifted_values : values; ++ int l = nearest_int(0.5f*(id*scales[ib]+127.f)); ++ l = std::max(0, std::min(127, l)) << 1; ++ y[ibl].scales[ib] |= l; ++ l -= 127; ++ float dl = d * l; ++ float idl = dl ? 1/dl : 0.f; ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ for (int j = 0; j < block_size; ++j) { ++ uint8_t idx = best_index_iq5nl(block_values, idl*xb[j]); ++ y[ibl].qs[block_size*(ib/2) + j] |= ((idx & 0xf) << 4*(ib%2)); ++ y[ibl].qh[j] |= ((idx >> 4) << ib); ++ float w = weight[j]; ++ float q = block_values[idx]*l; ++ sumqx += w*q*xb[j]; ++ sumq2 += w*q*q; ++ } ++ } ++ } ++ if (sumq2 > 0) *dptr = sumqx/sumq2; ++} ++} ++ ++void quantize_row_iq5_ks_ref(const float * x, block_iq5_ks * y, int64_t k) { ++ quantize_iq5_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq5_ks(const float * x, void * y, int64_t k) { ++ quantize_iq5_ks(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++size_t quantize_iq5_ks(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ5_KS, n_per_row); ++ float weight[kBlockSize]; ++ std::vector all_scales(n_per_row/kBlockSize); ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ auto q_func = [&all_scales, &weight, block_size = kBlockSize] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq5_ks_impl(QK_K, block_size, n_per_row, x, (char *)vy, all_scales.data(), weight, iq5nl_values, imatrix, 5); ++ }; ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq5_ks(const block_iq5_ks * x, float * y, int64_t k) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(k%QK_K == 0); ++ const float * dptr = (const float *)x; ++ float d = *dptr; ++ x = (const block_iq5_ks *)(dptr + 1); ++ int nblock = k/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto qs = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ for (int ib64 = 0; ib64 < QK_K/(2*kBlockSize); ++ib64) { ++ float dl1 = d * ((int)(x[ibl].scales[2*ib64+0] & 254) - 127); ++ float dl2 = d * ((int)(x[ibl].scales[2*ib64+1] & 254) - 127); ++ const int8_t * values1 = iq5nl_values + ((x[ibl].scales[2*ib64+0] & 1) << 5); ++ const int8_t * values2 = iq5nl_values + ((x[ibl].scales[2*ib64+1] & 1) << 5); ++ for (int j = 0; j < kBlockSize; ++j) { ++ y[j ] = dl1 * values1[(qs[j] & 0xf) | (((qh[j] >> (2*ib64+0)) & 1) << 4)]; ++ y[j+kBlockSize] = dl2 * values2[(qs[j] >> 4) | (((qh[j] >> (2*ib64+1)) & 1) << 4)]; ++ } ++ y += 2*kBlockSize; ++ qs += kBlockSize; ++ } ++ } ++} ++ ++void vec_dot_iq5_ks_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ constexpr int kBlockSize = 32; ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ5_KS, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ const float * dptr = (const float *)vx; ++ const float d = *dptr; ++ const block_iq5_ks * x = (const block_iq5_ks *)(dptr + 1); ++ const block_q8_K * y = (const block_q8_K *)vy; ++ int nblock = n/QK_K; ++ float sumf = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto qy = y[ibl].qs; ++ auto qs = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ float db = d * y[ibl].d; ++ for (int ib64 = 0; ib64 < QK_K/(2*kBlockSize); ++ib64) { ++ float dl1 = db * ((int)(x[ibl].scales[2*ib64+0] & 254) - 127); ++ float dl2 = db * ((int)(x[ibl].scales[2*ib64+1] & 254) - 127); ++ const int8_t * values1 = iq5nl_values + ((x[ibl].scales[2*ib64+0] & 1) << 5); ++ const int8_t * values2 = iq5nl_values + ((x[ibl].scales[2*ib64+1] & 1) << 5); ++ int suml1 = 0; ++ int suml2 = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ suml1 += qy[j ] * values1[(qs[j] & 0xf) | (((qh[j] >> (2*ib64+0)) & 1) << 4)]; ++ suml2 += qy[j+kBlockSize] * values2[(qs[j] >> 4) | (((qh[j] >> (2*ib64+1)) & 1) << 4)]; ++ } ++ sumf += dl1*suml1 + dl2*suml2; ++ y += 2*kBlockSize; ++ qs += kBlockSize; ++ } ++ } ++ *s = sumf; ++} ++ ++namespace { ++const uint16_t * scramble_table() { ++ static std::mutex mutex; ++ static std::vector table; ++ std::lock_guard lock(mutex); ++ if (table.empty()) { ++ table.resize(1 << 15); ++ for (int i = 0; i < int(table.size()); ++i) { ++ uint16_t val = i; ++ int non = popcount(val); ++ if (non%2) val |= (1 << 15); ++ bool found = false; ++ for (int j = 0; j < int(table.size()); ++j) { ++ if ((j ^ (j << 1)) == val) { ++ table[i] = j; found = true; break; ++ } ++ } ++ if (!found) { ++ printf("Oops: did not find for %d %u\n", i, val); ++ exit(1); ++ } ++ } ++ } ++ return table.data(); ++} ++uint16_t prune_iq4ks(uint16_t v, const int8_t * values, const float * x, const float * w, float dl) { ++ if (popcount(v)%2 == 0) return v; ++ float best_score = std::numeric_limits::max(); ++ uint8_t q4[4]; ++ int jbest = -1; ++ uint8_t bestq = 0; ++ for (int j = 0; j < 4; ++j) { ++ uint8_t q = (v >> 4*j) & 0xf; ++ q4[j] = q; ++ auto pc = popcount(q); ++ float diff0 = dl*iq4k_values[q] - x[j]; ++ int qmin = std::max(int(q)-2, 0); ++ int qmax = std::min(int(q)+2, 15); ++ for (int iq = qmin; iq <= qmax; ++iq) { ++ uint8_t qq = iq; ++ if (qq == q) continue; ++ int pci = popcount(qq); ++ if (std::abs(pci - pc)%2) { ++ float diff1 = dl*values[qq] - x[j]; ++ float score = w[j]*(diff1*diff1 - diff0*diff0); ++ if (score < best_score) { ++ best_score = score; jbest = j; bestq = qq; ++ } ++ } ++ } ++ } ++ GGML_ASSERT(jbest >= 0); ++ q4[jbest] = bestq; ++ return (q4[0] | (q4[1] << 4) | (q4[2] << 8) | (q4[3] << 12)); ++} ++static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy, ++ float * all_scales, float * weight, ++ const int8_t * values, ++ const float * quant_weights, ++ const uint16_t * table, ++ const int ntry) { ++ ++ constexpr int super_block_size = 256; ++ constexpr int block_size = 32; ++ ++ float * dptr = (float *)cy; ++ *dptr = 0; ++ block_iq4_kss * y = (block_iq4_kss *)(dptr + 1); ++ ++ const int8_t * shifted_values = values + 16; ++ ++ uint16_t vps[block_size/2], vms[block_size/2], vs[block_size/2]; ++ float xv[4], wv[4]; ++ ++ float amax_scale = 0; ++ ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ memset(&y[ibl], 0, sizeof(block_iq4_kss)); ++ const float * xbl = x + ibl*super_block_size; ++ auto scales = all_scales + ibl*(super_block_size/block_size); ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ float amax = 0, max = 0; ++ for (int j = 0; j < block_size; ++j) { ++ float ax = fabsf(xb[j]); ++ if (ax > amax) { ++ amax = ax; max = xb[j]; ++ } ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float best = 0; ++ float d = -max/iq4k_values[0]; ++ std::memset(vs, 0, block_size); ++ for (int itry = -ntry; itry <= ntry; ++itry) { ++ float id = (itry + values[0])/max; ++ float sumqx_p = 0, sumq2_p = 0; ++ float sumqx_m = 0, sumq2_m = 0; ++ float this_d = 1/id; ++ for (int k = 0; k < block_size/4; ++k) { ++ xv[0] = xb[2*k+0]; xv[1] = xb[2*k+0+block_size/2]; xv[2] = xb[2*k+1]; xv[3] = xb[2*k+1+block_size/2]; ++ wv[0] = weight[2*k+0]; wv[1] = weight[2*k+0+block_size/2]; wv[2] = weight[2*k+1]; wv[3] = weight[2*k+1+block_size/2]; ++ uint16_t vp = 0, vm = 0; ++ for (int j = 0; j < 4; ++j) { ++ float al = id*xv[j]; ++ vp |= (best_index_iq4nl(values, al) << 4*j); ++ vm |= (best_index_iq4nl(values, -al) << 4*j); ++ } ++ vp = prune_iq4ks(vp, values, xv, wv, this_d); ++ vm = prune_iq4ks(vm, values, xv, wv, this_d); ++ for (int j = 0; j < 4; ++j) { ++ float w = wv[j]; ++ float q = values[(vp >> 4*j) & 0xf]; ++ sumqx_p += w*q*xv[j]; ++ sumq2_p += w*q*q; ++ q = values[(vm >> 4*j) & 0xf]; ++ sumqx_m += w*q*xv[j]; ++ sumq2_m += w*q*q; ++ } ++ vps[k] = vp; ++ vms[k] = vm; ++ } ++ bool copy_p = false, copy_m = false; ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; copy_p = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; copy_m = true; ++ } ++ if (copy_m) { ++ std::memcpy(vs, vms, block_size); ++ } else if (copy_p) { ++ std::memcpy(vs, vps, block_size); ++ } ++ ++ id = (itry + shifted_values[0])/max; ++ this_d = 1/id; ++ sumqx_p = sumq2_p = 0; ++ sumqx_m = sumq2_m = 0; ++ for (int k = 0; k < block_size/4; ++k) { ++ xv[0] = xb[2*k+0]; xv[1] = xb[2*k+0+block_size/2]; xv[2] = xb[2*k+1]; xv[3] = xb[2*k+1+block_size/2]; ++ wv[0] = weight[2*k+0]; wv[1] = weight[2*k+0+block_size/2]; wv[2] = weight[2*k+1]; wv[3] = weight[2*k+1+block_size/2]; ++ uint16_t vp = 0, vm = 0; ++ for (int j = 0; j < 4; ++j) { ++ float al = id*xv[j]; ++ vp |= (best_index_iq4nl(shifted_values, al) << 4*j); ++ vm |= (best_index_iq4nl(shifted_values, -al) << 4*j); ++ } ++ vp = prune_iq4ks(vp, shifted_values, xv, wv, this_d); ++ vm = prune_iq4ks(vm, shifted_values, xv, wv, this_d); ++ for (int j = 0; j < 4; ++j) { ++ float w = wv[j]; ++ float q = shifted_values[(vp >> 4*j) & 0xf]; ++ sumqx_p += w*q*xv[j]; ++ sumq2_p += w*q*q; ++ q = shifted_values[(vm >> 4*j) & 0xf]; ++ sumqx_m += w*q*xv[j]; ++ sumq2_m += w*q*q; ++ } ++ vps[k] = vp; ++ vms[k] = vm; ++ } ++ copy_p = copy_m = false; ++ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) { ++ d = sumqx_p/sumq2_p; best = d * sumqx_p; copy_p = true; ++ } ++ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) { ++ d = sumqx_m/sumq2_m; best = d * sumqx_m; copy_m = true; ++ } ++ if (copy_m) { ++ std::memcpy(vs, vms, block_size); ++ } else if (copy_p) { ++ std::memcpy(vs, vps, block_size); ++ } ++ } ++ scales[ib] = d; ++ amax_scale = std::max(amax_scale, std::abs(d)); ++ } ++ } ++ float d = amax_scale/127; ++ *dptr = d; ++ if (!d) return; ++ float id = 1/d; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { ++ auto scales = all_scales + (super_block_size/block_size)*ibl; ++ const float * xbl = x + ibl*super_block_size; ++ float sigma2 = 0; ++ for (int j = 0; j < super_block_size; ++j) sigma2 += xbl[j]*xbl[j]; ++ sigma2 *= 2.f/super_block_size; ++ for (int ib = 0; ib < super_block_size/block_size; ++ib) { ++ const float * xb = xbl + ib*block_size; ++ if (quant_weights) { ++ const float * qw = quant_weights + ibl*super_block_size + ib*block_size; ++ for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ } else { ++ for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; ++ } ++ int l = nearest_int(0.5f*(id*scales[ib]+127.f)); ++ l = (std::max(0, std::min(127, l)) << 1) - 127; ++ if (l) { ++ float dl = d*l; ++ float idl = 1/dl; ++ float mse_p = 0, mse_m = 0; ++ for (int k = 0; k < block_size/4; ++k) { ++ xv[0] = xb[2*k+0]; xv[1] = xb[2*k+0+block_size/2]; xv[2] = xb[2*k+1]; xv[3] = xb[2*k+1+block_size/2]; ++ wv[0] = weight[2*k+0]; wv[1] = weight[2*k+0+block_size/2]; wv[2] = weight[2*k+1]; wv[3] = weight[2*k+1+block_size/2]; ++ uint16_t vp = 0, vm = 0; ++ for (int j = 0; j < 4; ++j) { ++ float al = idl*xv[j]; ++ vp |= (best_index_iq4nl( values, al) << 4*j); ++ vm |= (best_index_iq4nl(shifted_values, al) << 4*j); ++ } ++ vp = prune_iq4ks(vp, values, xv, wv, dl); ++ vm = prune_iq4ks(vm, shifted_values, xv, wv, dl); ++ for (int j = 0; j < 4; ++j) { ++ float w = wv[j]; ++ float q = values[(vp >> 4*j) & 0xf]; ++ mse_p += w*(xv[j] - dl*q)*(xv[j] - dl*q); ++ q = shifted_values[(vm >> 4*j) & 0xf]; ++ mse_m += w*(xv[j] - dl*q)*(xv[j] - dl*q); ++ } ++ vps[k] = vp; ++ vms[k] = vm; ++ } ++ const uint16_t * v = vps; ++ const int8_t * block_values = values; ++ if (mse_m < mse_p) { ++ v = vms; ++ block_values = values + 16; ++ } ++ for (int k = 0; k < block_size/4; ++k) { ++ xv[0] = xb[2*k+0]; xv[1] = xb[2*k+0+block_size/2]; xv[2] = xb[2*k+1]; xv[3] = xb[2*k+1+block_size/2]; ++ wv[0] = weight[2*k+0]; wv[1] = weight[2*k+0+block_size/2]; wv[2] = weight[2*k+1]; wv[3] = weight[2*k+1+block_size/2]; ++ for (int j = 0; j < 4; ++j) { ++ float q = block_values[(v[k] >> 4*j) & 0xf] * l; ++ sumqx += wv[j]*q*xv[j]; ++ sumq2 += wv[j]*q*q; ++ } ++ } ++ l += 127; ++ if (mse_m < mse_p) l |= 1; ++ uint16_t * q16 = (uint16_t *)y[ibl].qs + (block_size/4)*ib; ++ for (int k = 0; k < block_size/4; ++k) { ++ auto val = table[v[k] & 0x7fff]; ++ q16[k] = (val << 1) | ((l >> k) & 1); ++ } ++ } else { ++ l += 127; ++ uint16_t * q16 = (uint16_t *)y[ibl].qs + (block_size/4)*ib; ++ for (int k = 0; k < block_size/4; ++k) { ++ q16[k] = ((l >> k) & 1); ++ } ++ } ++ } ++ } ++ if (sumq2 > 0) *dptr = sumqx/sumq2 * 1.01f; ++} ++} ++ ++size_t quantize_iq4_kss(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_KSS, n_per_row); ++ std::vector all_scales(n_per_row/kBlockSize); ++ float weight[kBlockSize]; ++ auto table = scramble_table(); ++ QHelper helper(imatrix, user_data, n_per_row, kBlockSize); ++ auto q_func = [&all_scales, &weight, table] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ quantize_row_iq4_kss_impl(n_per_row, x, (char *)vy, all_scales.data(), weight, iq4k_values, imatrix, table, 7); ++ }; ++ helper.quantize(nrows, src, dst, row_size, q_func); ++ return nrows * row_size; ++} ++ ++void quantize_row_iq4_kss_ref(const float * x, block_iq4_kss * y, int64_t k) { ++ quantize_iq4_kss(x, y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_kss(const float * x, void * y, int64_t k) { ++ quantize_iq4_kss(x, (block_iq4_kss *)y, 1, k, nullptr, nullptr); ++} ++ ++void dequantize_row_iq4_kss(const block_iq4_kss * x, float * y, int64_t k) { ++ const float * dptr = (const float *)x; ++ const float d = *dptr; ++ x = (const block_iq4_kss *)(dptr + 1); ++ uint16_t aux16[8]; ++ const uint8_t * aux8 = (const uint8_t *)aux16; ++ for (int ibl = 0; ibl < k/QK_K; ++ibl) { ++ auto qs = (const uint16_t *)x[ibl].qs; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int16_t ls = 0; ++ for (int k = 0; k < 8; ++k) { ++ aux16[k] = qs[k] & 0xfffe; ++ aux16[k] ^= (aux16[k] >> 1); ++ ls |= (qs[k] & 1) << k; ++ } ++ const int8_t * values = iq4k_values + ((ls & 1) << 4); ++ float dl = d * ((ls & 254) - 127); ++ for (int j = 0; j < 16; ++j) { ++ y[j+ 0] = dl * values[aux8[j] & 0xf]; ++ y[j+16] = dl * values[aux8[j] >> 4]; ++ } ++ y += 32; ++ qs += 8; ++ } ++ } ++} ++ ++void vec_dot_iq4_kss_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_KSS, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK_K == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq4_nl_r4 ++// ++void quantize_row_iq4_nl_r4_ref(const float * x, block_iq4_nl_r4 * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_iq4_nl_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_nl_r4(const float * x, void * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_iq4_nl_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq4_nl(int nrows, int n_per_row, const block_iq4_nl * x, block_iq4_nl_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK4_NL == 0); ++ int nblock = n_per_row/QK4_NL; ++ const block_iq4_nl * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 4; ++k) y[ib].d[k] = x4[k][ib].d; ++ for (int k = 0; k < 4; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[4*k+i+ 0] = (x4[k][ib].qs[i+0] & 0xf) | ((x4[k][ib].qs[i+ 8] & 0x0f) << 4); // 0....3 + 8...11 from each row ++ y[ib].qs[4*k+i+16] = (x4[k][ib].qs[i+0] >> 4) | ((x4[k][ib].qs[i+ 8] & 0xf0)); // 16...19 + 24...27 from each row ++ y[ib].qs[4*k+i+32] = (x4[k][ib].qs[i+4] & 0xf) | ((x4[k][ib].qs[i+12] & 0x0f) << 4); // 4....7 + 12...15 from each row ++ y[ib].qs[4*k+i+48] = (x4[k][ib].qs[i+4] >> 4) | ((x4[k][ib].qs[i+12] & 0xf0)); // 20...23 + 28...31 from each row ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq4_nl_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ auto row_size_nl = ggml_row_size(GGML_TYPE_IQ4_NL, n_per_row); ++ std::vector qtmp(4*row_size_nl); ++ QHelper helper(imatrix, user_data, n_per_row, 32); ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq4_nl(x, (char *)vy, 1, n_per_row, imatrix, nullptr); ++ }; ++ char * qrow = (char *)dst; ++ for (int row = 0; row < nrows; row += 4) { ++ helper.quantize(4, src, qtmp.data(), row_size_nl, q_func); ++ repack_iq4_nl(4, n_per_row, (const block_iq4_nl *)qtmp.data(), (block_iq4_nl_r4 *)qrow, false); ++ src += 4*n_per_row; ++ qrow += 4*row_size_nl; ++ } ++ return nrows*row_size_nl; ++} ++ ++void dequantize_row_iq4_nl_r4(const block_iq4_nl_r4 * x, float * y, int64_t k) { ++ // we assume we are called with 4 rows ++ int n_per_row = k/4; ++ int nb = n_per_row/QK4_NL; ++ float * yk[4]; ++ for (int k = 0; k < 4; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ float scale = GGML_FP16_TO_FP32(x[ib].d[k]); ++ for (int i = 0; i < 4; ++i) { ++ yk[k][QK4_NL*ib+i+ 0] = scale * iq4k_values[x[ib].qs[4*k+i+ 0] & 0xf]; ++ yk[k][QK4_NL*ib+i+ 8] = scale * iq4k_values[x[ib].qs[4*k+i+ 0] >> 4]; ++ yk[k][QK4_NL*ib+i+16] = scale * iq4k_values[x[ib].qs[4*k+i+16] & 0xf]; ++ yk[k][QK4_NL*ib+i+24] = scale * iq4k_values[x[ib].qs[4*k+i+16] >> 4]; ++ yk[k][QK4_NL*ib+i+ 4] = scale * iq4k_values[x[ib].qs[4*k+i+32] & 0xf]; ++ yk[k][QK4_NL*ib+i+12] = scale * iq4k_values[x[ib].qs[4*k+i+32] >> 4]; ++ yk[k][QK4_NL*ib+i+20] = scale * iq4k_values[x[ib].qs[4*k+i+48] & 0xf]; ++ yk[k][QK4_NL*ib+i+28] = scale * iq4k_values[x[ib].qs[4*k+i+48] >> 4]; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq4_nl_r4_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_NL_R4, vx, 0, GGML_TYPE_Q8_0, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q4_0_r8 ++// ++void quantize_row_q4_0_r8_ref(const float * x, block_iq4_nl_r8 * y, int64_t k) { ++ // we assume we are called with 8 rows ++ quantize_q4_0_r8(x, (void *)y, 8, k/8, nullptr, nullptr); ++} ++ ++void quantize_row_q4_0_r8(const float * x, void * y, int64_t k) { ++ // we assume we are called with 8 rows ++ quantize_q4_0_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++static void repack_q4_0(int nrows, int n_per_row, const block_q4_0 * x, block_iq4_nl_r8 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%QK4_0 == 0); ++ int nblock = n_per_row/QK4_0; ++ const block_q4_0 * x8[8]; ++ for (int row = 0; row < nrows; row += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = x + nblock*k; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 8; ++k) { ++ y[ib].d[k] = x8[k][ib].d; ++ for (int l = 0; l < 4; ++l) { ++ for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i] = x8[k][ib].qs[4*l + i]; ++ } ++ } ++ } ++#ifdef __ARM_NEON ++ if (online) { ++ for (int l = 0; l < 8; ++l) { ++ auto v = vld1q_u8(y[ib].qs + 16*l); ++ vst1q_u8(y[ib].qs + 16*l, veorq_u8(v, vdupq_n_u8(0x88))); ++ } ++ } ++#endif ++ } ++ x += 8*nblock; ++ y += nblock; ++ } ++} ++#ifdef __ARM_NEON ++static void modify_q4_0_r8(int64_t k, char * cy) { ++ auto y = (block_iq4_nl_r8 *)cy; ++ int nb = k/(32*8); ++ for (int ib = 0; ib < nb; ++ib) { ++ auto v1 = vld1q_u8_x4(y[ib].qs); ++ auto v2 = vld1q_u8_x4(y[ib].qs+64); ++ for (int j = 0; j < 4; ++j) { ++ v1.val[j] = veorq_u8(v1.val[j], vdupq_n_u8(0x88)); ++ v2.val[j] = veorq_u8(v2.val[j], vdupq_n_u8(0x88)); ++ } ++ vst1q_u8_x4(y[ib].qs+ 0, v1); ++ vst1q_u8_x4(y[ib].qs+64, v2); ++ } ++} ++#endif ++ ++size_t quantize_q4_0_r8(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%8 == 0); ++ auto row_size_nl = ggml_row_size(GGML_TYPE_Q4_0, n_per_row); ++ std::vector qtmp(8*row_size_nl); ++ QHelper helper(imatrix, user_data, n_per_row, 32); ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q4_0(x, (char *)vy, 1, n_per_row, imatrix, nullptr); ++ }; ++ char * qrow = (char *)dst; ++ for (int row = 0; row < nrows; row += 8) { ++ helper.quantize(8, src, qtmp.data(), row_size_nl, q_func); ++ repack_q4_0(8, n_per_row, (const block_q4_0 *)qtmp.data(), (block_iq4_nl_r8 *)qrow, false); ++ src += 8*n_per_row; ++ qrow += 8*row_size_nl; ++ } ++ return nrows*row_size_nl; ++} ++ ++void dequantize_row_q4_0_r8(const block_iq4_nl_r8 * x, float * y, int64_t k) { ++ // we assume we are called with 8 rows ++ int n_per_row = k/8; ++ int nb = n_per_row/QK4_0; ++ float * yk[8]; ++ for (int k = 0; k < 8; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 8; ++k) { ++ float scale = GGML_FP16_TO_FP32(x[ib].d[k]); ++ for (int l = 0; l < 4; ++l) { ++ for (int i = 0; i < 4; ++i) { ++ yk[k][QK4_0*ib+4*l+i+ 0] = scale * ((x[ib].qs[32*l+4*k+i] & 0xf) - 8); ++ yk[k][QK4_0*ib+4*l+i+16] = scale * ((x[ib].qs[32*l+4*k+i] >> 4) - 8); ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q4_0_r8_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q4_0_R8, vx, 0, GGML_TYPE_Q8_0, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++ ++// ++// ========================================= q8_0_r8 ++// ++void quantize_row_q8_0_r8_ref(const float * x, block_q8_0_r8 * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q8_0_r8(x, (void *)y, 8, k/8, nullptr, nullptr); ++} ++ ++void quantize_row_q8_0_r8(const float * x, void * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q8_0_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++static void repack_q8_0(int nrows, int n_per_row, const block_q8_0 * x, block_q8_0_r8 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%QK8_0 == 0); ++ int nblock = n_per_row/QK8_0; ++ const block_q8_0 * x8[8]; ++ for (int row = 0; row < nrows; row += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = x + nblock*k; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 8; ++k) y[ib].d[k] = x8[k][ib].d; ++ for (int l = 0; l < 4; ++l) { ++ for (int k = 0; k < 8; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i+ 0] = x8[k][ib].qs[i+4*l+ 0]; ++ y[ib].qs[32*l+4*k+i+128] = x8[k][ib].qs[i+4*l+16]; ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ if (online) { ++ for (int l = 0; l < 4; ++l) { ++ auto v = _mm512_add_epi8(_mm512_loadu_si512((const __m512i *)y[ib].qs + l), _mm512_set1_epi8(127)); ++ _mm512_storeu_si512((__m512i *)y[ib].qs + l, v); ++ } ++ } ++#endif ++ } ++ x += 8*nblock; ++ y += nblock; ++ } ++} ++ ++#ifdef HAVE_FANCY_SIMD ++static void modify_q8_0_r8(int64_t k, char * cy) { ++ auto y = (block_q8_0_r8 *)cy; ++ int nb = k/(32*8); ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int l = 0; l < 4; ++l) { ++ auto v = _mm512_add_epi8(_mm512_loadu_si512((const __m512i *)y[ib].qs + l), _mm512_set1_epi8(127)); ++ _mm512_storeu_si512((__m512i *)y[ib].qs + l, v); ++ } ++ } ++} ++#endif ++ ++size_t quantize_q8_0_r8(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%8 == 0); ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q8_0, n_per_row); ++ std::vector qtmp(8*row_size_0); ++ char * qrow = (char *)dst; ++ for (int row = 0; row < nrows; row += 8) { ++ iqkbase_quantize_q8_0(src, qtmp.data(), 8, n_per_row, imatrix, nullptr); ++ repack_q8_0(8, n_per_row, (const block_q8_0 *)qtmp.data(), (block_q8_0_r8 *)qrow, false); ++ src += 8*n_per_row; ++ qrow += 8*row_size_0; ++ } ++ return nrows*row_size_0; ++} ++ ++void dequantize_row_q8_0_r8(const block_q8_0_r8 * x, float * y, int64_t k) { ++ // we assume we are called with 4 rows ++ int n_per_row = k/8; ++ int nb = n_per_row/QK8_0; ++ float * yk[8]; ++ for (int k = 0; k < 8; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 8; ++k) { ++ float scale = GGML_FP16_TO_FP32(x[ib].d[k]); ++ for (int l = 0; l < 4; ++l) for (int i = 0; i < 4; ++i) { ++ yk[k][QK8_0*ib+4*l+i+ 0] = scale * x[ib].qs[32*l+4*k+i+ 0]; ++ yk[k][QK8_0*ib+4*l+i+16] = scale * x[ib].qs[32*l+4*k+i+128]; ++ } ++ } ++ } ++} ++ ++void vec_dot_q8_0_r8_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q8_0_R8, vx, 0, GGML_TYPE_Q8_0, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q5_0_r4 ++// ++void quantize_row_q5_0_r4_ref(const float * x, block_q5_0_r4 * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q5_0_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q5_0_r4(const float * x, void * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q5_0_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static inline void convert_q5_0(const block_q5_0& x, uint8_t * L) { ++ uint32_t qh; ++ memcpy(&qh, x.qh, sizeof(qh)); ++ ++ for (int j = 0; j < QK5_0/2; ++j) { ++ const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10; ++ const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10; ++ ++ L[j ] = (x.qs[j] & 0x0F) | xh_0; ++ L[j + QK4_0/2] = (x.qs[j] >> 4) | xh_1; ++ } ++} ++ ++static void repack_q5_0(int nrows, int n_per_row, const block_q5_0 * x, block_q5_0_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK5_0 == 0); ++ int nblock = n_per_row/QK5_0; ++ const block_q5_0 * x4[4]; ++ uint8_t L[QK5_0]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ib = 0; ib < nblock; ++ib) { ++ std::memset(y[ib].qh, 0, QK5_0/2); ++ for (int k = 0; k < 4; ++k) { ++ y[ib].d[k] = x4[k][ib].d; ++ convert_q5_0(x4[k][ib], L); ++ for (int l = 0; l < 4; ++l) { ++ int l1 = 4*(l/2) + 16*(l%2), l2 = l1 + 8; ++ for (int i = 0; i < 4; ++i) { ++ y[ib].qs[4*k+i+16*l] = (L[i + l1] & 0xf) | ((L[i + l2] & 0xf) << 4); ++ y[ib].qh[4*k+i] |= ((L[i + l1] >> 4) | ((L[i + l2] >> 4) << 4)) << l; ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q5_0_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q5_0, n_per_row); ++ std::vector qtmp(4*row_size_0); ++ char * qrow = (char *)dst; ++ for (int row = 0; row < nrows; row += 4) { ++ iqkbase_quantize_q5_0(src, qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_q5_0(4, n_per_row, (const block_q5_0 *)qtmp.data(), (block_q5_0_r4 *)qrow, false); ++ src += 4*n_per_row; ++ qrow += 4*row_size_0; ++ } ++ return nrows*row_size_0; ++} ++ ++void dequantize_row_q5_0_r4(const block_q5_0_r4 * x, float * y, int64_t k) { ++ // we assume we are called with 4 rows ++ int n_per_row = k/4; ++ int nb = n_per_row/QK8_0; ++ float * yk[4]; ++ for (int k = 0; k < 4; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ float d = GGML_FP16_TO_FP32(x[ib].d[k]); ++ float m = -16*d; ++ for (int l = 0; l < 4; ++l) { ++ int ll = 16*(l%2) + 4*(l/2); ++ for (int i = 0; i < 4; ++i) { ++ yk[k][QK4_0*ib+i+ll+0] = d * ((x[ib].qs[4*k+i+16*l] & 0xf) | (((x[ib].qh[4*k+i] >> (l+0)) & 1) << 4)) + m; ++ yk[k][QK4_0*ib+i+ll+8] = d * ((x[ib].qs[4*k+i+16*l] >> 4) | (((x[ib].qh[4*k+i] >> (l+4)) & 1) << 4)) + m; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q5_0_r4_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q5_0_R4, vx, 0, GGML_TYPE_Q8_0, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q6_0_r4 ++// ++void quantize_row_q6_0_r4_ref(const float * x, block_q6_0_r4 * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q6_0_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q6_0_r4(const float * x, void * y, int64_t k) { ++ // we assume we are called with 4 rows ++ quantize_q6_0_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static inline void convert_q6_0(const block_q6_0& x, uint8_t * L) { ++ ++ for (int j = 0; j < QK6_0/2; ++j) { ++ const uint8_t h = x.qh[j%(QK6_0/4)] >> 4*(j/(QK6_0/4)); ++ L[j ] = (x.qs[j] & 0x0F) | ((h << 4) & 0x30); ++ L[j + QK6_0/2] = (x.qs[j] >> 4) | ((h << 2) & 0x30); ++ } ++} ++ ++static void repack_q6_0(int nrows, int n_per_row, const block_q6_0 * x, block_q6_0_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK5_0 == 0); ++ int nblock = n_per_row/QK6_0; ++ const block_q6_0 * x4[4]; ++ uint8_t L[QK6_0]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ib = 0; ib < nblock; ++ib) { ++ std::memset(y[ib].qh, 0, QK6_0); ++ for (int k = 0; k < 4; ++k) { ++ y[ib].d[k] = x4[k][ib].d; ++ convert_q6_0(x4[k][ib], L); ++ for (int l = 0; l < 4; ++l) { ++ int l1 = 4*(l/2) + 16*(l%2), l2 = l1 + 8; ++ for (int i = 0; i < 4; ++i) { ++ y[ib].qs[4*k+i+16*l] = (L[i + l1] & 0xf) | ((L[i + l2] & 0xf) << 4); ++ y[ib].qh[4*k+i+16*(l%2)] |= ((L[i + l1] >> 4) | ((L[i + l2] >> 4) << 4)) << 2*(l/2); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q6_0_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q6_0, n_per_row); ++ std::vector qtmp(4*row_size_0); ++ char * qrow = (char *)dst; ++ QHelper helper(imatrix, user_data, n_per_row, 32); ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q6_0(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ for (int row = 0; row < nrows; row += 4) { ++ helper.quantize(4, src, qtmp.data(), row_size_0, q_func); ++ repack_q6_0(4, n_per_row, (const block_q6_0 *)qtmp.data(), (block_q6_0_r4 *)qrow, false); ++ src += 4*n_per_row; ++ qrow += 4*row_size_0; ++ } ++ return nrows*row_size_0; ++} ++ ++void dequantize_row_q6_0_r4(const block_q6_0_r4 * x, float * y, int64_t k) { ++ // we assume we are called with 4 rows ++ int n_per_row = k/4; ++ int nb = n_per_row/QK6_0; ++ float * yk[4]; ++ for (int k = 0; k < 4; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ float d = GGML_FP16_TO_FP32(x[ib].d[k]); ++ float m = -32*d; ++ for (int l = 0; l < 4; ++l) { ++ int ll = 16*(l%2) + 4*(l/2); ++ for (int i = 0; i < 4; ++i) { ++ yk[k][QK4_0*ib+i+ll+0] = d * ((x[ib].qs[4*k+i+16*l] & 0xf) | (((x[ib].qh[4*k+i+16*(l%2)] >> (2*(l/2)+0)) & 3) << 4)) + m; ++ yk[k][QK4_0*ib+i+ll+8] = d * ((x[ib].qs[4*k+i+16*l] >> 4) | (((x[ib].qh[4*k+i+16*(l%2)] >> (2*(l/2)+4)) & 3) << 4)) + m; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q6_0_r4_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q6_0_R4, vx, 0, GGML_TYPE_Q8_0, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq4_xs_r8 ++// ++ ++void quantize_row_iq4_xs_r8_ref(const float * x, block_iq4_xs_r8 * y, int64_t k) { ++ quantize_iq4_xs_r8(x, (void *)y, 8, k/8, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_xs_r8(const float * x, void * y, int64_t k) { ++ quantize_iq4_xs_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++static void repack_iq4_xs(int nrows, int n_per_row, const block_iq4_xs * x, block_iq4_xs_r8 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq4_xs * x8[8]; ++ for (int row = 0; row < nrows; row += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 8; ++k) { ++ y[ibl].d[k] = x8[k][ibl].d; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ uint8_t sl = (x8[k][ibl].scales_l[ib/2] >> 4*(ib%2)) & 0xf; ++ uint8_t sh = (x8[k][ibl].scales_h >> 2*ib) & 3; ++ int i = 8*ib + k; ++ y[ibl].scales_l[i%32] |= (sl << 4*(i/32)); ++ y[ibl].scales_h[i%16] |= (sh << 2*(i/16)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[128*ib+4*k+i+ 0] = (x8[k][ibl].qs[16*ib+i+0] & 0xf) | ((x8[k][ibl].qs[16*ib+i+ 4] & 0xf) << 4); ++ y[ibl].qs[128*ib+4*k+i+32] = (x8[k][ibl].qs[16*ib+i+8] & 0xf) | ((x8[k][ibl].qs[16*ib+i+12] & 0xf) << 4); ++ y[ibl].qs[128*ib+4*k+i+64] = (x8[k][ibl].qs[16*ib+i+0] >> 4) | ((x8[k][ibl].qs[16*ib+i+ 4] >> 4) << 4); ++ y[ibl].qs[128*ib+4*k+i+96] = (x8[k][ibl].qs[16*ib+i+8] >> 4) | ((x8[k][ibl].qs[16*ib+i+12] >> 4) << 4); ++ } ++ } ++ } ++ } ++ x += 8*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq4_xs_r8(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq4_xs(x, (char *)vy, 1, n_per_row, imatrix, nullptr); ++ }; ++ return quantize_repack<32, block_iq4_xs, block_iq4_xs_r8, 8>(GGML_TYPE_IQ4_XS, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq4_xs); ++} ++ ++void dequantize_row_iq4_xs_r8(const block_iq4_xs_r8 * x, float * y, int64_t k) { ++ auto n_per_row = k/8; ++ float * y8[8]; ++ for (int k = 0; k < 8; ++k) y8[k] = y + n_per_row*k; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 8; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 3) << 4)) - 32); ++ for (int l = 0; l < 4; ++l) for (int i = 0; i < 4; ++i) { ++ y8[k][QK_K*ibl+32*ib+8*l+i+0] = dl * iq4k_values[x[ibl].qs[128*ib+4*k+i+32*l] & 0xf]; ++ y8[k][QK_K*ibl+32*ib+8*l+i+4] = dl * iq4k_values[x[ibl].qs[128*ib+4*k+i+32*l] >> 4]; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq4_xs_r8_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_XS_R8, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq4_ks_r4 ++// ++ ++void quantize_row_iq4_ks_r4_ref(const float * x, block_iq4_ks_r4 * y, int64_t k) { ++ quantize_iq4_ks_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_ks_r4(const float * x, void * y, int64_t k) { ++ quantize_iq4_ks_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq4_ks(int nrows, int n_per_row, const block_iq4_ks * x, block_iq4_ks_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_KS, n_per_row); ++ int nblock = n_per_row/QK_K; ++ char * cy = (char *)y; ++ const char * cx = (const char *)x; ++ const block_iq4_ks * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ float * dptr = (float *)cy; ++ block_iq4_ks_r4 * y = (block_iq4_ks_r4 *)(dptr + 4); ++ for (int k = 0; k < 4; ++k) { ++ auto dk = (const float *)(cx + k*row_size); ++ dptr[k] = dk[0]; ++ x4[k] = (const block_iq4_ks *)(dk + 1); ++ } ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales[4*ib+k] = x4[k][ibl].scales[ib]; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = (x4[k][ibl].qs[16*ib+i+0] & 0xf) | ((x4[k][ibl].qs[16*ib+i+ 8] & 0x0f) << 4); // 0....3 + 8...11 from each row ++ y[ibl].qs[64*ib+4*k+i+16] = (x4[k][ibl].qs[16*ib+i+0] >> 4) | ((x4[k][ibl].qs[16*ib+i+ 8] & 0xf0)); // 16...19 + 24...27 from each row ++ y[ibl].qs[64*ib+4*k+i+32] = (x4[k][ibl].qs[16*ib+i+4] & 0xf) | ((x4[k][ibl].qs[16*ib+i+12] & 0x0f) << 4); // 4....7 + 12...15 from each row ++ y[ibl].qs[64*ib+4*k+i+48] = (x4[k][ibl].qs[16*ib+i+4] >> 4) | ((x4[k][ibl].qs[16*ib+i+12] & 0xf0)); // 20...23 + 28...31 from each row ++ } ++ } ++ } ++ } ++ cx += 4*row_size; ++ cy += 4*row_size; ++ } ++} ++ ++size_t quantize_iq4_ks_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_KS, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq4_ks(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq4_ks(4, n_per_row, (const block_iq4_ks *)qtmp.data(), (block_iq4_ks_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq4_ks_r4(const block_iq4_ks_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ const float * dptr = (const float *)x; ++ x = (const block_iq4_ks_r4 *)(dptr + 4); ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = dptr[k]; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ float dl = d * ((x[ibl].scales[4*ib + k] & 254) - 127); ++ auto values = iq4k_values + ((x[ibl].scales[4*ib + k] & 1) << 4); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl * values[x[ibl].qs[64*ib+4*k+i+ 0] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl * values[x[ibl].qs[64*ib+4*k+i+ 0] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl * values[x[ibl].qs[64*ib+4*k+i+16] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl * values[x[ibl].qs[64*ib+4*k+i+16] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl * values[x[ibl].qs[64*ib+4*k+i+32] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl * values[x[ibl].qs[64*ib+4*k+i+32] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl * values[x[ibl].qs[64*ib+4*k+i+48] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl * values[x[ibl].qs[64*ib+4*k+i+48] >> 4]; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq4_ks_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_KS_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq2_bn_r4 ++// ++void quantize_row_iq2_bn_r4_ref(const float * x, block_iq2_bn * y, int64_t k) { ++ quantize_iq2_bn_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_bn_r4(const float * x, void * y, int64_t k) { ++ quantize_iq2_bn_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++void repack_iq2_bn(int nrows, int n_per_row, const char * x, char * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_IQ1BN == 0); ++ int nblock = n_per_row/QK_IQ1BN; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_BN, n_per_row); ++ const uint8_t * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ float * dr4 = (float *)(y + 4*row*row_size); ++ for (int k = 0; k < 4; ++k) { ++ const float * dptr = (const float *)(x + (row + k)*row_size); ++ dr4[k] = *dptr; ++ x4[k] = (const uint8_t *)(dptr + 1); ++ } ++ uint8_t * y4 = (uint8_t *)(dr4 + 4); ++ //std::memset(y4, 0, n_per_row); ++ for (int ib = 0; ib < nblock; ++ib) { ++ // 0...3 from rows 0...3 go to 1st 2 bits of 0...15 ++ // 16..19 from rows 0...3 go to 1st 2 bits of 16...31 ++ // 32..35 from rows 0...3 go to 1st 2 bits of 32...47 ++ // 48..51 from rows 0...3 go to 1st 2 bits of 48...63 ++ // 4...7 from rows 0...3 go to 2nd 2 bits of 0...15 ++ // 20..23 from rows 0...3 go to 2nd 2 bits of 16...31 ++ // 36..39 from rows 0...3 go to 2nd 2 bits of 32...47 ++ // 52..55 from rows 0...3 go to 2nd 2 bits of 48...63 ++ // 8..11 from rows 0...3 go to 3rd 2 bits of 0...15 ++ // 24..27 from rows 0...3 go to 3rd 2 bits of 16...31 ++ // 40..43 from rows 0...3 go to 3rd 2 bits of 32...47 ++ // 56..59 from rows 0...3 go to 3rd 2 bits of 48...63 ++ // 12..15 from rows 0...3 go to 4th 2 bits of 0...15 ++ // 28..31 from rows 0...3 go to 4th 2 bits of 16...31 ++ // 44..47 from rows 0...3 go to 4th 2 bits of 32...47 ++ // 60..63 from rows 0...3 go to 4th 2 bits of 48...63 ++ for (int k = 0; k < 4; ++k) { ++ for (int l = 0; l < 4; ++l) for (int i = 0; i < 4; ++i) { ++ y4[64*ib + 4*k + i + 16*l] = (((x4[k][16*ib + i + 0] >> 2*l) & 3) << 0) | ++ (((x4[k][16*ib + i + 4] >> 2*l) & 3) << 2) | ++ (((x4[k][16*ib + i + 8] >> 2*l) & 3) << 4) | ++ (((x4[k][16*ib + i + 12] >> 2*l) & 3) << 6); ++ //y4[64*ib + 4*k + i + 0] |= (x4[k][16*ib + i] >> 0) & 3; ++ //y4[64*ib + 4*k + i + 16] |= (x4[k][16*ib + i] >> 2) & 3; ++ //y4[64*ib + 4*k + i + 32] |= (x4[k][16*ib + i] >> 4) & 3; ++ //y4[64*ib + 4*k + i + 48] |= (x4[k][16*ib + i] >> 6) & 3; ++ //y4[64*ib + 4*k + i + 0] |= ((x4[k][16*ib + i + 4] >> 0) & 3) << 2; ++ //y4[64*ib + 4*k + i + 16] |= ((x4[k][16*ib + i + 4] >> 2) & 3) << 2; ++ //y4[64*ib + 4*k + i + 32] |= ((x4[k][16*ib + i + 4] >> 4) & 3) << 2; ++ //y4[64*ib + 4*k + i + 48] |= ((x4[k][16*ib + i + 4] >> 6) & 3) << 2; ++ //y4[64*ib + 4*k + i + 0] |= ((x4[k][16*ib + i + 8] >> 0) & 3) << 4; ++ //y4[64*ib + 4*k + i + 16] |= ((x4[k][16*ib + i + 8] >> 2) & 3) << 4; ++ //y4[64*ib + 4*k + i + 32] |= ((x4[k][16*ib + i + 8] >> 4) & 3) << 4; ++ //y4[64*ib + 4*k + i + 48] |= ((x4[k][16*ib + i + 8] >> 6) & 3) << 4; ++ //y4[64*ib + 4*k + i + 0] |= ((x4[k][16*ib + i + 12] >> 0) & 3) << 6; ++ //y4[64*ib + 4*k + i + 16] |= ((x4[k][16*ib + i + 12] >> 2) & 3) << 6; ++ //y4[64*ib + 4*k + i + 32] |= ((x4[k][16*ib + i + 12] >> 4) & 3) << 6; ++ //y4[64*ib + 4*k + i + 48] |= ((x4[k][16*ib + i + 12] >> 6) & 3) << 6; ++ } ++ } ++ } ++ } ++} ++} ++ ++size_t quantize_iq2_bn_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_IQ1BN == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_BN, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq2_bn(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq2_bn(4, n_per_row, qtmp.data(), qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq2_bn_r4(const block_iq2_bn * x, float * y, int64_t k) { ++ static_assert(QK_IQ1BN == 64); ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ const float * d4 = (const float *)x; ++ const uint8_t * qx = (const uint8_t *)(d4 + 4); ++ int nblock = n_per_row/QK_IQ1BN; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ for (int l = 0; l < 4; ++l) for (int i = 0; i < 4; ++i) { ++ uint8_t q = qx[4*k + i + 16*l]; ++ y4[k][64*ib + 16*l + i + 0] = d4[k] * (((q >> 0) & 3) - 1); ++ y4[k][64*ib + 16*l + i + 4] = d4[k] * (((q >> 2) & 3) - 1); ++ y4[k][64*ib + 16*l + i + 8] = d4[k] * (((q >> 4) & 3) - 1); ++ y4[k][64*ib + 16*l + i + 12] = d4[k] * (((q >> 6) & 3) - 1); ++ } ++ } ++ qx += 64; ++ } ++} ++ ++void vec_dot_iq2_bn_r4_q8_K64(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_BN_R4, vx, 0, GGML_TYPE_Q8_K64, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q4_k_r4 ++// ++ ++void quantize_row_q4_k_r4_ref(const float * x, block_q4_k_r4 * y, int64_t k) { ++ quantize_q4_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q4_k_r4(const float * x, void * y, int64_t k) { ++ quantize_q4_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t& d, uint8_t& m) { ++ if (j < 4) { ++ d = q[j] & 63; m = q[j + 4] & 63; ++ } else { ++ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); ++ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); ++ } ++} ++inline void convert_q4_k(const block_q4_K& x, uint8_t * L, uint8_t * Ld, uint8_t * Lm) { ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ get_scale_min_k4(2*ib64+0, x.scales, Ld[2*ib64+0], Lm[2*ib64+0]); ++ get_scale_min_k4(2*ib64+1, x.scales, Ld[2*ib64+1], Lm[2*ib64+1]); ++ for (int j = 0; j < 32; ++j) { ++ L[64*ib64+j+ 0] = x.qs[32*ib64+j] & 0xf; ++ L[64*ib64+j+32] = x.qs[32*ib64+j] >> 4; ++ } ++ } ++} ++} ++ ++static void repack_q4_k(int nrows, int n_per_row, const block_q4_K * x, block_q4_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q4_K * x4[4]; ++ uint8_t L[QK_K], Ld[QK_K/32], Lm[QK_K/32]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k+0] = x4[k][ibl].d; ++ y[ibl].d[k+4] = x4[k][ibl].dmin; ++ convert_q4_k(x4[k][ibl], L, Ld, Lm); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales_l[4*ib+k] = (Ld[ib] & 0xf) | ((Lm[ib] & 0xf) << 4); ++ uint8_t h = (Ld[ib] >> 4) | ((Lm[ib] >> 4) << 2); ++ y[ibl].scales_h[(4*ib+k)%16] |= (h << 4*((4*ib+k)/16)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = L[32*ib+i+ 0] | (L[32*ib+i+ 8] << 4); ++ y[ibl].qs[64*ib+4*k+i+16] = L[32*ib+i+16] | (L[32*ib+i+24] << 4); ++ y[ibl].qs[64*ib+4*k+i+32] = L[32*ib+i+ 4] | (L[32*ib+i+12] << 4); ++ y[ibl].qs[64*ib+4*k+i+48] = L[32*ib+i+20] | (L[32*ib+i+28] << 4); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q4_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q4_K(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<32, block_q4_K, block_q4_k_r4, 4>(GGML_TYPE_Q4_K, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_q4_k); ++} ++ ++void dequantize_row_q4_k_r4(const block_q4_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k+0]); ++ const float m = GGML_FP16_TO_FP32(x[ibl].d[k+4]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 4*ib + k; ++ float dl = d * ((x[ibl].scales_l[is] & 0xf) | (((x[ibl].scales_h[is%16] >> 4*(is/16)) & 0x03) << 4)); ++ float ml = m * ((x[ibl].scales_l[is] >> 4) | (((x[ibl].scales_h[is%16] >> 4*(is/16)) & 0x0c) << 2)); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl * (x[ibl].qs[64*ib+4*k+i+ 0] & 0xf) - ml; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl * (x[ibl].qs[64*ib+4*k+i+ 0] >> 4) - ml; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl * (x[ibl].qs[64*ib+4*k+i+16] & 0xf) - ml; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl * (x[ibl].qs[64*ib+4*k+i+16] >> 4) - ml; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl * (x[ibl].qs[64*ib+4*k+i+32] & 0xf) - ml; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl * (x[ibl].qs[64*ib+4*k+i+32] >> 4) - ml; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl * (x[ibl].qs[64*ib+4*k+i+48] & 0xf) - ml; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl * (x[ibl].qs[64*ib+4*k+i+48] >> 4) - ml; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q4_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q4_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q6_k_r4 ++// ++ ++void quantize_row_q6_k_r4_ref(const float * x, block_q6_k_r4 * y, int64_t k) { ++ quantize_q6_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q6_k_r4(const float * x, void * y, int64_t k) { ++ quantize_q6_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_q6_k(const block_q6_K& x, uint8_t * L) { ++ const uint8_t * ql = x.ql; ++ const uint8_t * qh = x.qh; ++ ++ for (int n = 0; n < QK_K; n += 128) { ++ for (int l = 0; l < 32; ++l) { ++ L[n + l + 0] = (ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4); ++ L[n + l + 32] = (ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4); ++ L[n + l + 64] = (ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4); ++ L[n + l + 96] = (ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4); ++ } ++ ql += 64; ++ qh += 32; ++ } ++} ++} ++ ++static void repack_q6_k(int nrows, int n_per_row, const block_q6_K * x, block_q6_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q6_K * x4[4]; ++ uint8_t L[QK_K]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ convert_q6_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales[8*ib+k+0] = x4[k][ibl].scales[2*ib+0]; ++ y[ibl].scales[8*ib+k+4] = x4[k][ibl].scales[2*ib+1]; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].ql[64*ib+4*k+i+ 0] = (L[32*ib+i+ 0] & 0xf) | ((L[32*ib+i+ 8] & 0xf) << 4); ++ y[ibl].ql[64*ib+4*k+i+16] = (L[32*ib+i+16] & 0xf) | ((L[32*ib+i+24] & 0xf) << 4); ++ y[ibl].ql[64*ib+4*k+i+32] = (L[32*ib+i+ 4] & 0xf) | ((L[32*ib+i+12] & 0xf) << 4); ++ y[ibl].ql[64*ib+4*k+i+48] = (L[32*ib+i+20] & 0xf) | ((L[32*ib+i+28] & 0xf) << 4); ++ y[ibl].qh[32*ib+4*k+i+ 0] = (L[32*ib+i+ 0] >> 4) | ((L[32*ib+i+ 8] >> 4) << 2) | ((L[32*ib+i+ 4] >> 4) << 4) | ((L[32*ib+i+12] >> 4) << 6); ++ y[ibl].qh[32*ib+4*k+i+16] = (L[32*ib+i+16] >> 4) | ((L[32*ib+i+24] >> 4) << 2) | ((L[32*ib+i+20] >> 4) << 4) | ((L[32*ib+i+28] >> 4) << 6); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q6_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q6_K(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_q6_K, block_q6_k_r4, 4>(GGML_TYPE_Q6_K, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_q6_k); ++} ++ ++void dequantize_row_q6_k_r4(const block_q6_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ auto ql = x[ibl].ql; ++ auto qh = x[ibl].qh; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ float dl1 = d * x[ibl].scales[8*ib+k+0]; ++ float dl2 = d * x[ibl].scales[8*ib+k+4]; ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * (((ql[4*k+i+ 0] & 0xf) | ((qh[4*k+i+ 0] << 4) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * (((ql[4*k+i+ 0] >> 4) | ((qh[4*k+i+ 0] << 2) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * (((ql[4*k+i+16] & 0xf) | ((qh[4*k+i+16] << 4) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * (((ql[4*k+i+16] >> 4) | ((qh[4*k+i+16] << 2) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * (((ql[4*k+i+32] & 0xf) | ((qh[4*k+i+ 0] >> 0) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * (((ql[4*k+i+32] >> 4) | ((qh[4*k+i+ 0] >> 2) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * (((ql[4*k+i+48] & 0xf) | ((qh[4*k+i+16] >> 0) & 0x30)) - 32); ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * (((ql[4*k+i+48] >> 4) | ((qh[4*k+i+16] >> 2) & 0x30)) - 32); ++ } ++ ql += 64; ++ qh += 32; ++ } ++ } ++ } ++} ++ ++void vec_dot_q6_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q6_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++ ++// ++// ========================================= q5_k_r4 ++// ++ ++void quantize_row_q5_k_r4_ref(const float * x, block_q5_k_r4 * y, int64_t k) { ++ quantize_q5_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q5_k_r4(const float * x, void * y, int64_t k) { ++ quantize_q5_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_q5_k(const block_q5_K& x, uint8_t * L, uint8_t * Ld, uint8_t * Lm) { ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ get_scale_min_k4(2*ib64+0, x.scales, Ld[2*ib64+0], Lm[2*ib64+0]); ++ get_scale_min_k4(2*ib64+1, x.scales, Ld[2*ib64+1], Lm[2*ib64+1]); ++ for (int j = 0; j < 32; ++j) { ++ L[64*ib64+j+ 0] = (x.qs[32*ib64+j] & 0xf) | (((x.qh[j] >> (2*ib64+0)) & 1) << 4); ++ L[64*ib64+j+32] = (x.qs[32*ib64+j] >> 4) | (((x.qh[j] >> (2*ib64+1)) & 1) << 4); ++ } ++ } ++} ++} ++ ++static void repack_q5_k(int nrows, int n_per_row, const block_q5_K * x, block_q5_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q5_K * x4[4]; ++ uint8_t L[QK_K], Ld[QK_K/32], Lm[QK_K/32]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k+0] = x4[k][ibl].d; ++ y[ibl].d[k+4] = x4[k][ibl].dmin; ++ convert_q5_k(x4[k][ibl], L, Ld, Lm); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales_l[4*ib+k] = (Ld[ib] & 0xf) | ((Lm[ib] & 0xf) << 4); ++ uint8_t h = (Ld[ib] >> 4) | ((Lm[ib] >> 4) << 2); ++ y[ibl].scales_h[(4*ib+k)%16] |= (h << 4*((4*ib+k)/16)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = (L[32*ib+i+ 0] & 0xf) | ((L[32*ib+i+ 8] & 0xf) << 4); ++ y[ibl].qs[64*ib+4*k+i+16] = (L[32*ib+i+16] & 0xf) | ((L[32*ib+i+24] & 0xf) << 4); ++ y[ibl].qs[64*ib+4*k+i+32] = (L[32*ib+i+ 4] & 0xf) | ((L[32*ib+i+12] & 0xf) << 4); ++ y[ibl].qs[64*ib+4*k+i+48] = (L[32*ib+i+20] & 0xf) | ((L[32*ib+i+28] & 0xf) << 4); ++ y[ibl].qh[16*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] >> 4) << 0) | ((L[32*ib+i+ 8] >> 4) << 1) | ((L[32*ib+i+ 4] >> 4) << 2) | ((L[32*ib+i+12] >> 4) << 3) | ++ ((L[32*ib+i+16] >> 4) << 4) | ((L[32*ib+i+24] >> 4) << 5) | ((L[32*ib+i+20] >> 4) << 6) | ((L[32*ib+i+28] >> 4) << 7); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q5_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q5_K(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<32, block_q5_K, block_q5_k_r4, 4>(GGML_TYPE_Q5_K, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_q5_k); ++} ++ ++void dequantize_row_q5_k_r4(const block_q5_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k+0]); ++ const float m = GGML_FP16_TO_FP32(x[ibl].d[k+4]); ++ auto ql = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 4*ib + k; ++ float dl = d * ((x[ibl].scales_l[is] & 0xf) | (((x[ibl].scales_h[is%16] >> 4*(is/16)) & 0x03) << 4)); ++ float ml = m * ((x[ibl].scales_l[is] >> 4) | (((x[ibl].scales_h[is%16] >> 4*(is/16)) & 0x0c) << 2)); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl * ((ql[4*k+i+ 0] & 0xf) | ((qh[4*k+i] << 4) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl * ((ql[4*k+i+ 0] >> 4) | ((qh[4*k+i] << 3) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl * ((ql[4*k+i+16] & 0xf) | ((qh[4*k+i] >> 0) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl * ((ql[4*k+i+16] >> 4) | ((qh[4*k+i] >> 1) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl * ((ql[4*k+i+32] & 0xf) | ((qh[4*k+i] << 2) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl * ((ql[4*k+i+32] >> 4) | ((qh[4*k+i] << 1) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl * ((ql[4*k+i+48] & 0xf) | ((qh[4*k+i] >> 2) & 0x10)) - ml; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl * ((ql[4*k+i+48] >> 4) | ((qh[4*k+i] >> 3) & 0x10)) - ml; ++ } ++ ql += 64; ++ qh += 16; ++ } ++ } ++ } ++} ++ ++void vec_dot_q5_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q5_K_R4, vx, 0, GGML_TYPE_Q8_K32, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q3_k_r4 ++// ++ ++void quantize_row_q3_k_r4_ref(const float * x, block_q3_k_r4 * y, int64_t k) { ++ quantize_q3_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q3_k_r4(const float * x, void * y, int64_t k) { ++ quantize_q3_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_q3_k(const block_q3_K& x, uint8_t * L, uint8_t * Ld) { ++ constexpr uint32_t kmask1 = 0x03030303; ++ constexpr uint32_t kmask2 = 0x0f0f0f0f; ++ uint32_t aux[4]; ++ memcpy(aux, x.scales, 12); ++ uint32_t tmp = aux[2]; ++ aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); ++ aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); ++ aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); ++ aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); ++ std::memcpy(Ld, aux, 16); ++ ++ const uint8_t * q = x.qs; ++ const uint8_t * hm = x.hmask; ++ uint8_t m = 1; ++ for (int n = 0; n < QK_K; n += 128) { ++ int shift = 0; ++ for (int j = 0; j < 4; ++j) { ++ for (int l = 0; l < 32; ++l) { ++ *L++ = ((q[l] >> shift) & 3) + ((hm[l] & m) ? 4 : 0); ++ } ++ shift += 2; ++ m <<= 1; ++ } ++ q += 32; ++ } ++} ++} ++ ++static void repack_q3_k(int nrows, int n_per_row, const block_q3_K * x, block_q3_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q3_K * x4[4]; ++ uint8_t L[QK_K], Ld[QK_K/16]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ convert_q3_k(x4[k][ibl], L, Ld); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib+k; ++ y[ibl].scales_l[is%32] |= (Ld[2*ib+0] & 0xf) << 4*(is/32); ++ y[ibl].scales_h[is%16] |= (Ld[2*ib+0] >> 4) << 2*(is/16); ++ is += 4; ++ y[ibl].scales_l[is%32] |= (Ld[2*ib+1] & 0xf) << 4*(is/32); ++ y[ibl].scales_h[is%16] |= (Ld[2*ib+1] >> 4) << 2*(is/16); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[32*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] & 0x3) << 0) | ((L[32*ib+i+ 4] & 0x3) << 2) | ((L[32*ib+i+ 8] & 0x3) << 4) | ((L[32*ib+i+12] & 0x3) << 6); ++ y[ibl].qs[32*ib+4*k+i+16] = ((L[32*ib+i+16] & 0x3) << 0) | ((L[32*ib+i+20] & 0x3) << 2) | ((L[32*ib+i+24] & 0x3) << 4) | ((L[32*ib+i+28] & 0x3) << 6); ++ y[ibl].qh[16*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] >> 2) << 0) | ((L[32*ib+i+ 4] >> 2) << 1) | ((L[32*ib+i+ 8] >> 2) << 2) | ((L[32*ib+i+12] >> 2) << 3) ++ | ((L[32*ib+i+16] >> 2) << 4) | ((L[32*ib+i+20] >> 2) << 5) | ((L[32*ib+i+24] >> 2) << 6) | ((L[32*ib+i+28] >> 2) << 7); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q3_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q3_K(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_q3_K, block_q3_k_r4, 4>(GGML_TYPE_Q3_K, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_q3_k); ++} ++ ++void dequantize_row_q3_k_r4(const block_q3_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ auto ql = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl1 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 0x03) << 4)) - 32); ++ is += 4; ++ float dl2 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 0x03) << 4)) - 32); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * ((((ql[4*k+i+ 0] >> 0) & 3) | ((qh[4*k+i] << 2) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * ((((ql[4*k+i+ 0] >> 2) & 3) | ((qh[4*k+i] << 1) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * ((((ql[4*k+i+ 0] >> 4) & 3) | ((qh[4*k+i] << 0) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * ((((ql[4*k+i+ 0] >> 6) & 3) | ((qh[4*k+i] >> 1) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * ((((ql[4*k+i+16] >> 0) & 3) | ((qh[4*k+i] >> 2) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * ((((ql[4*k+i+16] >> 2) & 3) | ((qh[4*k+i] >> 3) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * ((((ql[4*k+i+16] >> 4) & 3) | ((qh[4*k+i] >> 4) & 4)) - 4); ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * ((((ql[4*k+i+16] >> 6) & 3) | ((qh[4*k+i] >> 5) & 4)) - 4); ++ } ++ ql += 32; ++ qh += 16; ++ } ++ } ++ } ++} ++ ++void vec_dot_q3_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q3_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q2_k_r4 ++// ++ ++void quantize_row_q2_k_r4_ref(const float * x, block_q2_k_r4 * y, int64_t k) { ++ quantize_q3_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_q2_k_r4(const float * x, void * y, int64_t k) { ++ quantize_q2_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_q2_k(const block_q2_K& x, uint8_t * L) { ++ ++ const uint8_t * qs = x.qs; ++ for (int n = 0; n < QK_K; n += 128) { ++ for (int j = 0; j < 32; ++j) { ++ L[n + j + 0] = (qs[j] >> 0) & 0x3; ++ L[n + j + 32] = (qs[j] >> 2) & 0x3; ++ L[n + j + 64] = (qs[j] >> 4) & 0x3; ++ L[n + j + 96] = (qs[j] >> 6) & 0x3; ++ } ++ qs += 32; ++ } ++} ++} ++ ++static void repack_q2_k(int nrows, int n_per_row, const block_q2_K * x, block_q2_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q2_K * x4[4]; ++ uint8_t L[QK_K]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k+0] = x4[k][ibl].d; ++ y[ibl].d[k+4] = x4[k][ibl].dmin; ++ for (int ib = 0; ib < QK_K/16; ++ib) { ++ y[ibl].scales[4*ib+k] = x4[k][ibl].scales[ib]; ++ } ++ convert_q2_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[32*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] & 0x3) << 0) | ((L[32*ib+i+ 4] & 0x3) << 2) | ((L[32*ib+i+ 8] & 0x3) << 4) | ((L[32*ib+i+12] & 0x3) << 6); ++ y[ibl].qs[32*ib+4*k+i+16] = ((L[32*ib+i+16] & 0x3) << 0) | ((L[32*ib+i+20] & 0x3) << 2) | ((L[32*ib+i+24] & 0x3) << 4) | ((L[32*ib+i+28] & 0x3) << 6); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q2_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_q2_K(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_q2_K, block_q2_k_r4, 4>(GGML_TYPE_Q2_K, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_q2_k); ++} ++ ++void dequantize_row_q2_k_r4(const block_q2_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k+0]); ++ const float m = GGML_FP16_TO_FP32(x[ibl].d[k+4]); ++ auto ql = x[ibl].qs; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ float dl1 = d * (x[ibl].scales[8*ib + k + 0] & 0xf); ++ float ml1 = m * (x[ibl].scales[8*ib + k + 0] >> 4); ++ float dl2 = d * (x[ibl].scales[8*ib + k + 4] & 0xf); ++ float ml2 = m * (x[ibl].scales[8*ib + k + 4] >> 4); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * ((ql[4*k+i+ 0] >> 0) & 3) - ml1; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * ((ql[4*k+i+ 0] >> 2) & 3) - ml1; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * ((ql[4*k+i+ 0] >> 4) & 3) - ml1; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * ((ql[4*k+i+ 0] >> 6) & 3) - ml1; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * ((ql[4*k+i+16] >> 0) & 3) - ml2; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * ((ql[4*k+i+16] >> 2) & 3) - ml2; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * ((ql[4*k+i+16] >> 4) & 3) - ml2; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * ((ql[4*k+i+16] >> 6) & 3) - ml2; ++ } ++ ql += 32; ++ } ++ } ++ } ++} ++ ++void vec_dot_q2_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q2_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq4_k_r4 ++// ++ ++void quantize_row_iq4_k_r4_ref(const float * x, block_iq4_k_r4 * y, int64_t k) { ++ quantize_iq4_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_k_r4(const float * x, void * y, int64_t k) { ++ quantize_iq4_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq4_k(int nrows, int n_per_row, const block_iq4_k * x, block_iq4_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq4_k * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].extra, 0, 8); ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ auto extra = x4[k][ibl].extra; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ if (extra & 1) y[ibl].extra[k+0] |= (1 << ib); ++ if (extra & 2) y[ibl].extra[k+4] |= (1 << ib); ++ extra >>= 2; ++ uint8_t sl1 = x4[k][ibl].scales_l[ib] & 0xf; ++ uint8_t sl2 = x4[k][ibl].scales_l[ib] >> 4; ++ uint8_t sh = x4[k][ibl].scales_h[ib/2] >> 4*(ib%2); ++ uint8_t sh1 = (sh >> 0) & 3; ++ uint8_t sh2 = (sh >> 2) & 3; ++ int i = 8*ib + k; ++ y[ibl].scales_l[i%32] |= (sl1 << 4*(i/32)); ++ y[ibl].scales_h[i%16] |= (sh1 << 2*(i/16)); ++ i += 4; ++ y[ibl].scales_l[i%32] |= (sl2 << 4*(i/32)); ++ y[ibl].scales_h[i%16] |= (sh2 << 2*(i/16)); ++ } ++ } ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ for (int k = 0; k < 4; ++k) for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = (x4[k][ibl].qs[16*ib+i+0] & 0xf) | ((x4[k][ibl].qs[16*ib+i+ 8] & 0x0f) << 4); // 0....3 + 8...11 from each row ++ y[ibl].qs[64*ib+4*k+i+16] = (x4[k][ibl].qs[16*ib+i+0] >> 4) | ((x4[k][ibl].qs[16*ib+i+ 8] & 0xf0)); // 16...19 + 24...27 from each row ++ y[ibl].qs[64*ib+4*k+i+32] = (x4[k][ibl].qs[16*ib+i+4] & 0xf) | ((x4[k][ibl].qs[16*ib+i+12] & 0x0f) << 4); // 4....7 + 12...15 from each row ++ y[ibl].qs[64*ib+4*k+i+48] = (x4[k][ibl].qs[16*ib+i+4] >> 4) | ((x4[k][ibl].qs[16*ib+i+12] & 0xf0)); // 20...23 + 28...31 from each row ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq4_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_K, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq4_k(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq4_k(4, n_per_row, (const block_iq4_k *)qtmp.data(), (block_iq4_k_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq4_k_r4(const block_iq4_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl1 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 3) << 4)) - 32); ++ is += 4; ++ float dl2 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 3) << 4)) - 32); ++ auto values1 = iq4k_values + (x[ibl].extra[k+0] & (1 << ib) ? 16 : 0); ++ auto values2 = iq4k_values + (x[ibl].extra[k+4] & (1 << ib) ? 16 : 0); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * values1[x[ibl].qs[64*ib+4*k+i+ 0] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * values1[x[ibl].qs[64*ib+4*k+i+ 0] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * values2[x[ibl].qs[64*ib+4*k+i+16] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * values2[x[ibl].qs[64*ib+4*k+i+16] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * values1[x[ibl].qs[64*ib+4*k+i+32] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * values1[x[ibl].qs[64*ib+4*k+i+32] >> 4]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * values2[x[ibl].qs[64*ib+4*k+i+48] & 0xf]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * values2[x[ibl].qs[64*ib+4*k+i+48] >> 4]; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq4_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq5_k_r4 ++// ++ ++void quantize_row_iq5_k_r4_ref(const float * x, block_iq5_k_r4 * y, int64_t k) { ++ quantize_iq5_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq5_k_r4(const float * x, void * y, int64_t k) { ++ quantize_iq5_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++template ++inline void convert_iq5_k(const Block& x, uint8_t * L) { ++ const uint8_t * qs = x.qs; ++ const uint8_t * qh = x.qh; ++ int shift = 0; ++ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { ++ for (int j = 0; j < 16; ++j) { ++ L[j+ 0] = (qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4); ++ L[j+16] = (qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4); ++ L[j+32] = (qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3); ++ L[j+48] = (qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3); ++ } ++ L += 64; ++ qs += 32; ++ shift += 2; ++ if (shift == 8) { qh += 32; shift = 0; } ++ } ++} ++} ++ ++static void repack_iq5_k(int nrows, int n_per_row, const block_iq5_k * x, block_iq5_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq5_k * x4[4]; ++ uint8_t L[QK_K]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].extra, 0, 8); ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/16); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ auto extra = x4[k][ibl].extra; ++ convert_iq5_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ if (extra & 1) y[ibl].extra[k+0] |= (1 << ib); ++ if (extra & 2) y[ibl].extra[k+4] |= (1 << ib); ++ extra >>= 2; ++ uint8_t sl1 = x4[k][ibl].scales_l[ib] & 0xf; ++ uint8_t sl2 = x4[k][ibl].scales_l[ib] >> 4; ++ uint8_t sh = x4[k][ibl].scales_h[ib/2] >> 4*(ib%2); ++ uint8_t sh1 = (sh >> 0) & 3; ++ uint8_t sh2 = (sh >> 2) & 3; ++ int i = 8*ib + k; ++ y[ibl].scales_l[i%32] |= (sl1 << 4*(i/32)); ++ y[ibl].scales_h[i%16] |= (sh1 << 2*(i/16)); ++ i += 4; ++ y[ibl].scales_l[i%32] |= (sl2 << 4*(i/32)); ++ y[ibl].scales_h[i%16] |= (sh2 << 2*(i/16)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = (L[32*ib+i+ 0] & 0xf) | ((L[32*ib+i+ 8] & 0xf) << 4); // 0....3 + 8...11 from each row ++ y[ibl].qs[64*ib+4*k+i+16] = (L[32*ib+i+16] & 0xf) | ((L[32*ib+i+24] & 0xf) << 4); // 16...19 + 24...27 from each row ++ y[ibl].qs[64*ib+4*k+i+32] = (L[32*ib+i+ 4] & 0xf) | ((L[32*ib+i+12] & 0xf) << 4); // 4....7 + 12...15 from each row ++ y[ibl].qs[64*ib+4*k+i+48] = (L[32*ib+i+20] & 0xf) | ((L[32*ib+i+28] & 0xf) << 4); // 20...23 + 28...31 from each row ++ y[ibl].qh[16*ib+4*k+i ] = ((L[32*ib+i+ 0] >> 4) << 0) | ((L[32*ib+i+ 8] >> 4) << 1) | ((L[32*ib+i+16] >> 4) << 2) | ((L[32*ib+i+24] >> 4) << 3) ++ | ((L[32*ib+i+ 4] >> 4) << 4) | ((L[32*ib+i+12] >> 4) << 5) | ((L[32*ib+i+20] >> 4) << 6) | ((L[32*ib+i+28] >> 4) << 7); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq5_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ5_K, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq5_k(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq5_k(4, n_per_row, (const block_iq5_k *)qtmp.data(), (block_iq5_k_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq5_k_r4(const block_iq5_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl1 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 3) << 4)) - 32); ++ is += 4; ++ float dl2 = d * ((((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) | (((x[ibl].scales_h[is%16] >> 2*(is/16)) & 3) << 4)) - 32); ++ auto values1 = iq5nl_values + (x[ibl].extra[k+0] & (1 << ib) ? 32 : 0); ++ auto values2 = iq5nl_values + (x[ibl].extra[k+4] & (1 << ib) ? 32 : 0); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * values1[(x[ibl].qs[64*ib+4*k+i+ 0] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 0) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * values1[(x[ibl].qs[64*ib+4*k+i+ 0] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 1) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * values2[(x[ibl].qs[64*ib+4*k+i+16] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 2) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * values2[(x[ibl].qs[64*ib+4*k+i+16] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 3) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * values1[(x[ibl].qs[64*ib+4*k+i+32] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 4) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * values1[(x[ibl].qs[64*ib+4*k+i+32] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 5) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * values2[(x[ibl].qs[64*ib+4*k+i+48] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 6) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * values2[(x[ibl].qs[64*ib+4*k+i+48] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 7) & 1) << 4)]; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq5_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ5_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq5_ks_r4 ++// ++ ++void quantize_row_iq5_ks_r4_ref(const float * x, block_iq5_ks_r4 * y, int64_t k) { ++ quantize_iq5_ks_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq5_ks_r4(const float * x, void * y, int64_t k) { ++ quantize_iq5_ks_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq5_ks(int nrows, int n_per_row, const block_iq5_ks * x, block_iq5_ks_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ5_KS, n_per_row); ++ int nblock = n_per_row/QK_K; ++ const block_iq5_ks * x4[4]; ++ uint8_t L[QK_K]; ++ char * cy = (char *)y; ++ const char * cx = (const char *)x; ++ for (int row = 0; row < nrows; row += 4) { ++ float * dptr = (float *)cy; ++ block_iq5_ks_r4 * y = (block_iq5_ks_r4 *)(dptr + 4); ++ for (int k = 0; k < 4; ++k) { ++ auto dk = (const float *)(cx + k*row_size); ++ dptr[k] = dk[0]; ++ x4[k] = (const block_iq5_ks *)(dk + 1); ++ } ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ convert_iq5_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales[4*ib+k] = x4[k][ibl].scales[ib]; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[64*ib+4*k+i+ 0] = (L[32*ib+i+ 0] & 0xf) | ((L[32*ib+i+ 8] & 0xf) << 4); // 0....3 + 8...11 from each row ++ y[ibl].qs[64*ib+4*k+i+16] = (L[32*ib+i+16] & 0xf) | ((L[32*ib+i+24] & 0xf) << 4); // 16...19 + 24...27 from each row ++ y[ibl].qs[64*ib+4*k+i+32] = (L[32*ib+i+ 4] & 0xf) | ((L[32*ib+i+12] & 0xf) << 4); // 4....7 + 12...15 from each row ++ y[ibl].qs[64*ib+4*k+i+48] = (L[32*ib+i+20] & 0xf) | ((L[32*ib+i+28] & 0xf) << 4); // 20...23 + 28...31 from each row ++ y[ibl].qh[16*ib+4*k+i ] = ((L[32*ib+i+ 0] >> 4) << 0) | ((L[32*ib+i+ 8] >> 4) << 1) | ((L[32*ib+i+16] >> 4) << 2) | ((L[32*ib+i+24] >> 4) << 3) ++ | ((L[32*ib+i+ 4] >> 4) << 4) | ((L[32*ib+i+12] >> 4) << 5) | ((L[32*ib+i+20] >> 4) << 6) | ((L[32*ib+i+28] >> 4) << 7); ++ } ++ } ++ } ++ } ++ cx += 4*row_size; ++ cy += 4*row_size; ++ } ++} ++ ++size_t quantize_iq5_ks_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ5_KS, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq5_ks(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq5_ks(4, n_per_row, (const block_iq5_ks *)qtmp.data(), (block_iq5_ks_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq5_ks_r4(const block_iq5_ks_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ //auto row_size = ggml_row_size(GGML_TYPE_IQ5_KS, n_per_row); ++ int nblock = n_per_row/QK_K; ++ const float * dptr = (const float *)x; ++ x = (const block_iq5_ks_r4 *)(dptr + 4); ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = dptr[k]; ++ //if (!isfinite(d)) { ++ // printf("Oops: d = %g for ibl = %d, k = %d\n", d, ibl, k); exit(1); ++ //} ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ uint8_t sc = x[ibl].scales[4*ib+k]; ++ float dl = d * ((sc & 254) - 127); ++ //if (!isfinite(dl)) { ++ // printf("Oops: dl = %g for ibl = %d, k = %d, ib = %d, d = %g, sc = %u\n", dl, ibl, k, ib, d, sc); exit(1); ++ //} ++ auto values = iq5nl_values + ((sc & 1) << 5); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl * values[(x[ibl].qs[64*ib+4*k+i+ 0] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 0) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl * values[(x[ibl].qs[64*ib+4*k+i+ 0] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 1) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl * values[(x[ibl].qs[64*ib+4*k+i+16] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 2) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl * values[(x[ibl].qs[64*ib+4*k+i+16] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 3) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl * values[(x[ibl].qs[64*ib+4*k+i+32] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 4) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl * values[(x[ibl].qs[64*ib+4*k+i+32] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 5) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl * values[(x[ibl].qs[64*ib+4*k+i+48] & 0xf) | (((x[ibl].qh[16*ib+4*k+i] >> 6) & 1) << 4)]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl * values[(x[ibl].qs[64*ib+4*k+i+48] >> 4) | (((x[ibl].qh[16*ib+4*k+i] >> 7) & 1) << 4)]; ++ } ++ //for (int i = 0; i < 32; ++i) { ++ // if (!isfinite(y4[k][QK_K*ibl+32*ib+i])) { ++ // printf("Oops: y4[%d][%d, %d, %d] = %g\n", k, ibl, ib, i, y4[k][QK_K*ibl+32*ib+i]); ++ // printf("d = %g, dl = %g\n", d, dl); ++ // exit(1); ++ // } ++ //} ++ } ++ } ++ } ++} ++ ++void vec_dot_iq5_ks_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ5_KS_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q8_k_r8 ++// ++ ++void quantize_row_q8_k_r8_ref(const float * x, block_q8_k_r8 * y, int64_t k) { ++ quantize_q8_k_r8(x, (void *)y, 8, k/8, nullptr, nullptr); ++} ++ ++void quantize_row_q8_k_r8(const float * x, void * y, int64_t k) { ++ quantize_q8_k_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++static void repack_q8_k(int nrows, int n_per_row, const block_q8_K * x, block_q8_k_r8 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q8_K * x8[8]; ++ for (int row = 0; row < nrows; row += 8) { ++ for (int k = 0; k < 8; ++k) x8[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 8; ++k) { ++ y[ibl].d[k] = GGML_FP32_TO_FP16(x8[k][ibl].d); ++ for (int ib = 0; ib < QK_K/4; ++ib) { ++ for (int i = 0; i < 4; ++i) y[ibl].qs[32*ib + 4*k + i] = x8[k][ibl].qs[4*ib+i]; ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ if (online) { ++ for (int l = 0; l < 32; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[ibl].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[ibl].qs + l, v); ++ } ++ } ++#endif ++ } ++ x += 8*nblock; ++ y += nblock; ++ } ++} ++#ifdef HAVE_FANCY_SIMD ++static void modify_q8_k_r8(int64_t k, char * cy) { ++ auto y = (block_q8_k_r8 *)cy; ++ int nb = k/(256*8); ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int l = 0; l < 32; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[ib].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[ib].qs + l, v); ++ } ++ } ++} ++#endif ++ ++size_t quantize_q8_k_r8(const float * src, void * dst, int64_t nrows, int64_t n_per_row, [[maybe_unused]] const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q8_K, n_per_row); ++ auto row_size_1 = ggml_row_size(GGML_TYPE_Q8_K_R8, n_per_row); ++ std::vector qtmp(8*row_size_0); ++ for (int row = 0; row < nrows; row += 8) { ++ quantize_row_q8_K32(src, (void *)qtmp.data(), 8*n_per_row); ++ repack_q8_k(8, n_per_row, (const block_q8_K *)qtmp.data(), (block_q8_k_r8 *)qcur, false); ++ qcur += 8*row_size_1; ++ src += 8*n_per_row; ++ } ++ return nrows*row_size_1; ++} ++ ++void dequantize_row_q8_k_r8(const block_q8_k_r8 * x, float * y, int64_t k) { ++ auto n_per_row = k/8; ++ float * y8[8]; ++ for (int k = 0; k < 8; ++k) y8[k] = y + n_per_row*k; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 8; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/4; ++ib) { ++ for (int i = 0; i < 4; ++i) { ++ y8[k][QK_K*ibl+4*ib+i] = d * x[ibl].qs[32*ib+4*k+i]; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q8_k_r8_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q8_K_R8, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q8_k_r16 ++// ++ ++void quantize_row_q8_k_r16_ref(const float * x, block_q8_k_r16 * y, int64_t k) { ++ quantize_q8_k_r16(x, (void *)y, 16, k/16, nullptr, nullptr); ++} ++ ++void quantize_row_q8_k_r16(const float * x, void * y, int64_t k) { ++ quantize_q8_k_r16(x, y, 16, k/16, nullptr, nullptr); ++} ++ ++static void repack_q16_k(int nrows, int n_per_row, const block_q8_K * x, block_q8_k_r16 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%16 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_q8_K * x16[16]; ++ for (int row = 0; row < nrows; row += 16) { ++ for (int k = 0; k < 16; ++k) x16[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 16; ++k) { ++ y[ibl].d[k] = GGML_FP32_TO_FP16(x16[k][ibl].d); ++ for (int ib = 0; ib < QK_K/4; ++ib) { ++ for (int i = 0; i < 4; ++i) y[ibl].qs[64*ib + 4*k + i] = x16[k][ibl].qs[4*ib+i]; ++ } ++ } ++#ifdef HAVE_FANCY_SIMD ++ for (int l = 0; l < 64; ++l) { ++ auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[ibl].qs + l), _mm512_set1_epi8(-128)); ++ _mm512_storeu_si512((__m512i *)y[ibl].qs + l, v); ++ } ++#endif ++ } ++ x += 16*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_q8_k_r16(const float * src, void * dst, int64_t nrows, int64_t n_per_row, [[maybe_unused]] const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%16 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q8_K, n_per_row); ++ auto row_size_1 = ggml_row_size(GGML_TYPE_Q8_K_R16, n_per_row); ++ std::vector qtmp(16*row_size_0); ++ for (int row = 0; row < nrows; row += 16) { ++ quantize_row_q8_K32(src, (void *)qtmp.data(), 16*n_per_row); ++ repack_q16_k(16, n_per_row, (const block_q8_K *)qtmp.data(), (block_q8_k_r16 *)qcur, false); ++ qcur += 16*row_size_1; ++ src += 16*n_per_row; ++ } ++ return nrows*row_size_1; ++} ++ ++void dequantize_row_q8_k_r16(const block_q8_k_r16 * x, float * y, int64_t k) { ++ auto n_per_row = k/16; ++ float * y16[16]; ++ for (int k = 0; k < 16; ++k) y16[k] = y + n_per_row*k; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto qs = (const uint8_t *)x[ibl].qs; ++ for (int k = 0; k < 16; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ const float m = -128.f*d; ++ for (int ib = 0; ib < QK_K/4; ++ib) { ++ for (int i = 0; i < 4; ++i) { ++ y16[k][QK_K*ibl+4*ib+i] = d * qs[64*ib+4*k+i] + m; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_q8_k_r16_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q8_K_R16, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= q8_KV_r8 ++// ++ ++void quantize_row_q8_KV_r8_ref(const float * x, void * y, int64_t k) { ++ quantize_q8_KV_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++void quantize_row_q8_KV_r8(const float * x, void * y, int64_t k) { ++ quantize_q8_KV_r8(x, y, 8, k/8, nullptr, nullptr); ++} ++ ++static void repack_q8_KV(int nrows, int n_per_row, const char * cx, char * cy, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%16 == 0); ++ auto row_size_x = ggml_row_size(GGML_TYPE_Q8_KV, n_per_row); ++ auto row_size_y = ggml_row_size(GGML_TYPE_Q8_KV_R8, n_per_row); ++ const int8_t * x8[8]; ++#ifdef __ARM_NEON ++ int8x16x2_t m0, m1, m2, m3; ++#endif ++ for (int row = 0; row < nrows; row += 8) { ++ auto dy = (float *)cy; ++ auto qy = (int8_t *)(dy + 8); ++ for (int k = 0; k < 8; ++k) { ++ auto dx = (const float *)(cx + k*row_size_x); ++ dy[k] = dx[0]; ++ x8[k] = (const int8_t *)(dx + 2); ++ } ++ for (int ib = 0; ib < n_per_row/16; ++ib) { ++#ifdef __AVX2__ ++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) ++ auto m0 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[4]+ib), _mm_loadu_si128((const __m128i *)x8[0]+ib)); ++ auto m1 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[5]+ib), _mm_loadu_si128((const __m128i *)x8[1]+ib)); ++ auto m2 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[6]+ib), _mm_loadu_si128((const __m128i *)x8[2]+ib)); ++ auto m3 = MM256_SET_M128I(_mm_loadu_si128((const __m128i *)x8[7]+ib), _mm_loadu_si128((const __m128i *)x8[3]+ib)); ++ auto t0 = _mm256_unpacklo_epi32(m0, m1); ++ auto t1 = _mm256_unpacklo_epi32(m2, m3); ++ auto t2 = _mm256_unpackhi_epi32(m0, m1); ++ auto t3 = _mm256_unpackhi_epi32(m2, m3); ++ m0 = _mm256_unpacklo_epi64(t0, t1); ++ m1 = _mm256_unpackhi_epi64(t0, t1); ++ m2 = _mm256_unpacklo_epi64(t2, t3); ++ m3 = _mm256_unpackhi_epi64(t2, t3); ++#ifdef HAVE_FANCY_SIMD ++ if (online) { ++ m0 = _mm256_add_epi8(m0, _mm256_set1_epi8(127)); ++ m1 = _mm256_add_epi8(m1, _mm256_set1_epi8(127)); ++ m2 = _mm256_add_epi8(m2, _mm256_set1_epi8(127)); ++ m3 = _mm256_add_epi8(m3, _mm256_set1_epi8(127)); ++ } ++#endif ++ _mm256_storeu_si256((__m256i *)qy + 4*ib+0, m0); ++ _mm256_storeu_si256((__m256i *)qy + 4*ib+1, m1); ++ _mm256_storeu_si256((__m256i *)qy + 4*ib+2, m2); ++ _mm256_storeu_si256((__m256i *)qy + 4*ib+3, m3); ++#elif defined __ARM_NEON ++ m0.val[0] = vld1q_s8(x8[0]+16*ib); m0.val[1] = vld1q_s8(x8[4]+16*ib); ++ m1.val[0] = vld1q_s8(x8[1]+16*ib); m1.val[1] = vld1q_s8(x8[5]+16*ib); ++ m2.val[0] = vld1q_s8(x8[2]+16*ib); m2.val[1] = vld1q_s8(x8[6]+16*ib); ++ m3.val[0] = vld1q_s8(x8[3]+16*ib); m3.val[1] = vld1q_s8(x8[7]+16*ib); ++ auto row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[0]), vreinterpretq_s32_s8(m1.val[0])); ++ auto row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[0]), vreinterpretq_s32_s8(m3.val[0])); ++ m0.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[0] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[0] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ row01 = vtrnq_s32(vreinterpretq_s32_s8(m0.val[1]), vreinterpretq_s32_s8(m1.val[1])); ++ row23 = vtrnq_s32(vreinterpretq_s32_s8(m2.val[1]), vreinterpretq_s32_s8(m3.val[1])); ++ m0.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m1.val[1] = vreinterpretq_s8_s64(vtrn1q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ m2.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[0]), vreinterpretq_s64_s32(row23.val[0]))); ++ m3.val[1] = vreinterpretq_s8_s64(vtrn2q_s64(vreinterpretq_s64_s32(row01.val[1]), vreinterpretq_s64_s32(row23.val[1]))); ++ vst1q_s8_x2(qy + 0 + 128*ib, m0); ++ vst1q_s8_x2(qy + 32 + 128*ib, m1); ++ vst1q_s8_x2(qy + 64 + 128*ib, m2); ++ vst1q_s8_x2(qy + 96 + 128*ib, m3); ++#else ++ // TODO ++ for (int l = 0; l < 4; ++l) { ++ for (int k = 0; k < 8; ++k) for (int i = 0; i < 4; ++i) { ++ y[ib].qs[32*l+4*k+i+ 0] = x8[k][ib].qs[i+4*l+ 0]; ++ y[ib].qs[32*l+4*k+i+128] = x8[k][ib].qs[i+4*l+16]; ++ } ++ } ++#endif ++ ++ } ++ cx += 8*row_size_x; ++ cy += online ? 8*row_size_x : 8*row_size_y; ++ //So, if we are run-time-repacking (online = true) we don't want to change the stride, so we just leave some unused space at the end of each row ++ } ++} ++#ifdef HAVE_FANCY_SIMD ++static void modify_q8_KV_r8(int64_t k, char * cy) { ++ int8_t * q8 = (int8_t *)(cy + 8*sizeof(float)); ++ for (int j = 0; j < k; ++j) q8[j] += 127; ++} ++#endif ++ ++size_t quantize_q8_KV_r8(const float * src, void * dst, int64_t nrows, int64_t n_per_row, [[maybe_unused]] const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%8 == 0); ++ GGML_ASSERT(n_per_row%16 == 0); ++ char * qcur = (char *)dst; ++ auto row_size_0 = ggml_row_size(GGML_TYPE_Q8_KV, n_per_row); ++ auto row_size_1 = ggml_row_size(GGML_TYPE_Q8_KV_R8, n_per_row); ++ std::vector qtmp(8*row_size_0); ++ for (int row = 0; row < nrows; row += 8) { ++ quantize_q8_KV(src, (void *)qtmp.data(), 8, n_per_row, imatrix, user_data); ++ repack_q8_KV(8, n_per_row, qtmp.data(), qcur, false); ++ qcur += 8*row_size_1; ++ src += 8*n_per_row; ++ } ++ return nrows*row_size_1; ++} ++ ++void dequantize_row_q8_KV_r8(const void * vx, float * y, int64_t k) { ++ auto n_per_row = k/8; ++ float * y8[8]; ++ for (int k = 0; k < 8; ++k) y8[k] = y + n_per_row*k; ++ auto dptr = (const float *)vx; ++ auto q8 = (const int8_t *)(dptr + 8); ++ for (int ib = 0; ib < n_per_row/16; ++ib) { ++ for (int k = 0; k < 8; ++k) { ++ for (int l = 0; l < 4; ++l) { ++ for (int i = 0; i < 4; ++i) y8[k][16*ib + 4*l + i] = dptr[k] * q8[128*ib + 32*l + 4*k + i]; ++ } ++ } ++ } ++} ++ ++void vec_dot_q8_KV_r8_q8_KV(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q8_KV_R8, vx, 0, GGML_TYPE_Q8_KV, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= bf16_r4 ++// ++namespace { ++inline ggml_bf16_t to_bf16(const float& x) { ++ union { float f; uint32_t u; } helper; ++ helper.f = x; ++ return ggml_bf16_t{(uint16_t)(helper.u >> 16)}; ++} ++inline ggml_bf16_t to_bf16(const ggml_half& x) { return to_bf16(GGML_FP16_TO_FP32(x)); } ++inline ggml_bf16_t to_bf16(const ggml_bf16_t& x) { return x; } ++template ++void repack_bf16(int nrows, int n_per_row, const T * x, ggml_bf16_t * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%16 == 0); ++ GGML_ASSERT(n_per_row%2 == 0); ++ for (int row = 0; row < nrows; row += 16) { ++ for (int k = 0; k < 16; ++k) { ++ auto x8 = x + k*n_per_row; ++ for (int ib = 0; ib < n_per_row/2; ++ib) { ++ y[32*ib + 2*k + 0] = to_bf16(x8[2*ib+0]); ++ y[32*ib + 2*k + 1] = to_bf16(x8[2*ib+1]); ++ } ++ } ++ x += 16*n_per_row; ++ y += 16*n_per_row; ++ } ++} ++} ++ ++void repack_f32_bf16_r16(const void * src, void * dst, int64_t nrows, int64_t n_per_row) { ++ repack_bf16(nrows, n_per_row, (const float *)src, (ggml_bf16_t *)dst, false); ++} ++ ++void repack_bf16_bf16_r16(const void * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row) { ++ repack_bf16(nrows, n_per_row, (const ggml_bf16_t *)src, (ggml_bf16_t *)dst, false); ++} ++ ++// ++// ========================================= iq3_k_r4 ++// ++ ++void quantize_row_iq3_k_r4_ref(const float * x, block_iq3_k_r4 * y, int64_t k) { ++ quantize_iq3_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_k_r4(const float * x, void * y, int64_t k) { ++ quantize_iq3_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_iq3_k(const block_iq3_k& x, uint8_t * L) { ++ const uint8_t * qs = x.qs; ++ const uint8_t * qh = x.qh; ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ int shift_l = 2*(ib32%4); ++ int shift_h = ib32%8; ++ for (int j = 0; j < 16; ++j) { ++ L[j+ 0] = ((qs[j+ 0] >> shift_l) & 3) | (((qh[j+ 0] >> shift_h) & 1) << 2); ++ L[j+16] = ((qs[j+16] >> shift_l) & 3) | (((qh[j+16] >> shift_h) & 1) << 2); ++ } ++ L += 32; ++ if (shift_l == 6) qs += 32; ++ } ++} ++} ++ ++static void repack_iq3_k(int nrows, int n_per_row, const block_iq3_k * x, block_iq3_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq3_k * x4[4]; ++ uint8_t L[QK_K]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].extra, 0, 8); ++ std::memset(y[ibl].scales_l, 0, QK_K/8); ++ std::memset(y[ibl].scales_h, 0, QK_K/32); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ auto extra = x4[k][ibl].extra; ++ uint16_t sh = x4[k][ibl].scales_h; ++ convert_iq3_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ if (extra & 1) y[ibl].extra[k+0] |= (1 << ib); ++ if (extra & 2) y[ibl].extra[k+4] |= (1 << ib); ++ extra >>= 2; ++ uint8_t sl1 = x4[k][ibl].scales_l[ib] & 0xf; ++ uint8_t sl2 = x4[k][ibl].scales_l[ib] >> 4; ++ uint8_t sh1 = (sh >> 0) & 1; ++ uint8_t sh2 = (sh >> 1) & 1; ++ sh >>= 2; ++ int i = 8*ib + k; ++ y[ibl].scales_l[i%32] |= (sl1 << 4*(i/32)); ++ y[ibl].scales_h[i%8 ] |= (sh1 << (i/8)); ++ i += 4; ++ y[ibl].scales_l[i%32] |= (sl2 << 4*(i/32)); ++ y[ibl].scales_h[i%8 ] |= (sh2 << (i/8)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[32*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] & 0x3) << 0) | ((L[32*ib+i+ 4] & 0x3) << 2) | ((L[32*ib+i+ 8] & 0x3) << 4) | ((L[32*ib+i+12] & 0x3) << 6); ++ y[ibl].qs[32*ib+4*k+i+16] = ((L[32*ib+i+16] & 0x3) << 0) | ((L[32*ib+i+20] & 0x3) << 2) | ((L[32*ib+i+24] & 0x3) << 4) | ((L[32*ib+i+28] & 0x3) << 6); ++ y[ibl].qh[16*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] >> 2) << 0) | ((L[32*ib+i+ 4] >> 2) << 1) | ((L[32*ib+i+ 8] >> 2) << 2) | ((L[32*ib+i+12] >> 2) << 3) ++ | ((L[32*ib+i+16] >> 2) << 4) | ((L[32*ib+i+20] >> 2) << 5) | ((L[32*ib+i+24] >> 2) << 6) | ((L[32*ib+i+28] >> 2) << 7); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq3_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ3_K, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq3_k(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq3_k(4, n_per_row, (const block_iq3_k *)qtmp.data(), (block_iq3_k_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq3_k_r4(const block_iq3_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ auto ql = x[ibl].qs; ++ auto qh = x[ibl].qh; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl1 = d * (2*((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) + 1) * ((x[ibl].scales_h[is%8] >> (is/8)) & 1 ? -1 : 1); ++ is += 4; ++ float dl2 = d * (2*((x[ibl].scales_l[is%32] >> 4*(is/32)) & 0xf) + 1) * ((x[ibl].scales_h[is%8] >> (is/8)) & 1 ? -1 : 1); ++ auto values1 = iq3nl_values + (x[ibl].extra[k+0] & (1 << ib) ? 8 : 0); ++ auto values2 = iq3nl_values + (x[ibl].extra[k+4] & (1 << ib) ? 8 : 0); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * values1[((ql[4*k+i+ 0] >> 0) & 3) | ((qh[4*k+i] << 2) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * values1[((ql[4*k+i+ 0] >> 2) & 3) | ((qh[4*k+i] << 1) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * values1[((ql[4*k+i+ 0] >> 4) & 3) | ((qh[4*k+i] << 0) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * values1[((ql[4*k+i+ 0] >> 6) & 3) | ((qh[4*k+i] >> 1) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * values2[((ql[4*k+i+16] >> 0) & 3) | ((qh[4*k+i] >> 2) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * values2[((ql[4*k+i+16] >> 2) & 3) | ((qh[4*k+i] >> 3) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * values2[((ql[4*k+i+16] >> 4) & 3) | ((qh[4*k+i] >> 4) & 4)]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * values2[((ql[4*k+i+16] >> 6) & 3) | ((qh[4*k+i] >> 5) & 4)]; ++ } ++ ql += 32; ++ qh += 16; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq3_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq2_k_r4 ++// ++ ++void quantize_row_iq2_k_r4_ref(const float * x, block_iq2_k_r4 * y, int64_t k) { ++ quantize_iq2_k_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_k_r4(const float * x, void * y, int64_t k) { ++ quantize_iq2_k_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++inline void convert_iq2_k(const block_iq2_k& x, uint8_t * L) { ++ const uint8_t * qs = x.qs; ++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) { ++ int shift_l = 2*(ib32%4); ++ for (int j = 0; j < 16; ++j) { ++ L[j+ 0] = ((qs[j+ 0] >> shift_l) & 3); ++ L[j+16] = ((qs[j+16] >> shift_l) & 3); ++ } ++ L += 32; ++ if (shift_l == 6) qs += 32; ++ } ++} ++} ++ ++static void repack_iq2_k(int nrows, int n_per_row, const block_iq2_k * x, block_iq2_k_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq2_k * x4[4]; ++ uint8_t L[QK_K]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].extra, 0, 8); ++ std::memset(y[ibl].scales, 0, QK_K/8); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ auto extra = x4[k][ibl].extra; ++ convert_iq2_k(x4[k][ibl], L); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ if (extra & 1) y[ibl].extra[k+0] |= (1 << ib); ++ if (extra & 2) y[ibl].extra[k+4] |= (1 << ib); ++ extra >>= 2; ++ uint8_t sl1 = x4[k][ibl].scales[ib] & 0xf; ++ uint8_t sl2 = x4[k][ibl].scales[ib] >> 4; ++ int i = 8*ib + k; ++ y[ibl].scales[i%32] |= (sl1 << 4*(i/32)); ++ i += 4; ++ y[ibl].scales[i%32] |= (sl2 << 4*(i/32)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[32*ib+4*k+i+ 0] = ((L[32*ib+i+ 0] & 0x3) << 0) | ((L[32*ib+i+ 4] & 0x3) << 2) | ((L[32*ib+i+ 8] & 0x3) << 4) | ((L[32*ib+i+12] & 0x3) << 6); ++ y[ibl].qs[32*ib+4*k+i+16] = ((L[32*ib+i+16] & 0x3) << 0) | ((L[32*ib+i+20] & 0x3) << 2) | ((L[32*ib+i+24] & 0x3) << 4) | ((L[32*ib+i+28] & 0x3) << 6); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq2_k_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ char * qcur = (char *)dst; ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_K, n_per_row); ++ std::vector qtmp(4*row_size); ++ for (int row = 0; row < nrows; row += 4) { ++ quantize_iq2_k(src, (void *)qtmp.data(), 4, n_per_row, imatrix, user_data); ++ repack_iq2_k(4, n_per_row, (const block_iq2_k *)qtmp.data(), (block_iq2_k_r4 *)qcur, false); ++ qcur += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq2_k_r4(const block_iq2_k_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ auto ql = x[ibl].qs; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int is = 8*ib + k; ++ float dl1 = d * (((x[ibl].scales[is%32] >> 4*(is/32)) & 0xf) - 8); ++ is += 4; ++ float dl2 = d * (((x[ibl].scales[is%32] >> 4*(is/32)) & 0xf) - 8); ++ auto values1 = iq2nl_values + (x[ibl].extra[k+0] & (1 << ib) ? 4 : 0); ++ auto values2 = iq2nl_values + (x[ibl].extra[k+4] & (1 << ib) ? 4 : 0); ++ for (int i = 0; i < 4; ++i) { ++ y4[k][QK_K*ibl+32*ib+i+ 0] = dl1 * values1[(ql[4*k+i+ 0] >> 0) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+ 4] = dl1 * values1[(ql[4*k+i+ 0] >> 2) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+ 8] = dl1 * values1[(ql[4*k+i+ 0] >> 4) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+12] = dl1 * values1[(ql[4*k+i+ 0] >> 6) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+16] = dl2 * values2[(ql[4*k+i+16] >> 0) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+20] = dl2 * values2[(ql[4*k+i+16] >> 2) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+24] = dl2 * values2[(ql[4*k+i+16] >> 4) & 3]; ++ y4[k][QK_K*ibl+32*ib+i+28] = dl2 * values2[(ql[4*k+i+16] >> 6) & 3]; ++ } ++ ql += 32; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq2_k_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_K_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++namespace { ++inline uint8_t scrambled_sign(uint8_t s) { ++ static const uint8_t k_table[128] = { ++ 0x00, 0x7f, 0x7e, 0x01, 0x7c, 0x03, 0x02, 0x7d, 0x78, 0x07, 0x06, 0x79, 0x04, 0x7b, 0x7a, 0x05, ++ 0x70, 0x0f, 0x0e, 0x71, 0x0c, 0x73, 0x72, 0x0d, 0x08, 0x77, 0x76, 0x09, 0x74, 0x0b, 0x0a, 0x75, ++ 0x60, 0x1f, 0x1e, 0x61, 0x1c, 0x63, 0x62, 0x1d, 0x18, 0x67, 0x66, 0x19, 0x64, 0x1b, 0x1a, 0x65, ++ 0x10, 0x6f, 0x6e, 0x11, 0x6c, 0x13, 0x12, 0x6d, 0x68, 0x17, 0x16, 0x69, 0x14, 0x6b, 0x6a, 0x15, ++ 0x40, 0x3f, 0x3e, 0x41, 0x3c, 0x43, 0x42, 0x3d, 0x38, 0x47, 0x46, 0x39, 0x44, 0x3b, 0x3a, 0x45, ++ 0x30, 0x4f, 0x4e, 0x31, 0x4c, 0x33, 0x32, 0x4d, 0x48, 0x37, 0x36, 0x49, 0x34, 0x4b, 0x4a, 0x35, ++ 0x20, 0x5f, 0x5e, 0x21, 0x5c, 0x23, 0x22, 0x5d, 0x58, 0x27, 0x26, 0x59, 0x24, 0x5b, 0x5a, 0x25, ++ 0x50, 0x2f, 0x2e, 0x51, 0x2c, 0x53, 0x52, 0x2d, 0x28, 0x57, 0x56, 0x29, 0x54, 0x2b, 0x2a, 0x55, ++ }; ++ return k_table[s]; ++} ++} ++ ++// ++// ========================================= iq2_xxs_r4 ++// ++ ++void quantize_row_iq2_xxs_r4_ref(const float * x, block_iq2_xxs_r4 * y, int64_t k) { ++ quantize_iq2_xxs_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_xxs_r4(const float * x, void * y, int64_t k) { ++ quantize_iq2_xxs_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq2_xxs(int nrows, int n_per_row, const block_iq2_xxs * x, block_iq2_xxs_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq2_xxs * x4[4]; ++ uint32_t aux32[2]; ++ const uint8_t * aux8 = (const uint8_t *)aux32; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto ysas = (uint32_t *)y[ibl].sas; ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ std::memcpy(aux32, x4[k][ibl].qs + 4*ib, 2*sizeof(uint32_t)); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[16*ib+4*k+i] = aux8[i]; ++ } ++ uint8_t scale = aux32[1] >> 28; ++ uint8_t s1 = (scrambled_sign((aux32[1] >> 0) & 127) << 1) | ((scale >> 0) & 1); ++ uint8_t s2 = (scrambled_sign((aux32[1] >> 7) & 127) << 1) | ((scale >> 1) & 1); ++ uint8_t s3 = (scrambled_sign((aux32[1] >> 14) & 127) << 1) | ((scale >> 2) & 1); ++ uint8_t s4 = (scrambled_sign((aux32[1] >> 21) & 127) << 1) | ((scale >> 3) & 1); ++ aux32[1] = uint32_t(s1) | (uint32_t(s2) << 8) | (uint32_t(s3) << 16) | (uint32_t(s4) << 24); ++ ysas[4*ib+k] = aux32[1]; ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq2_xxs_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq2_xxs(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<32, block_iq2_xxs, block_iq2_xxs_r4, 4>(GGML_TYPE_IQ2_XXS, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq2_xxs); ++} ++ ++void dequantize_row_iq2_xxs_r4(const block_iq2_xxs_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ uint32_t s32; ++ const uint8_t * s8 = (const uint8_t *)&s32; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const uint32_t * sas = (const uint32_t *)x[ibl].sas; ++ for (int k = 0; k < 4; ++k) { ++ const float d = 0.125f*GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ uint32_t aux32 = sas[4*ib+k]; ++ s32 = aux32 & 0x01010101; ++ uint8_t scale = s8[0] | (s8[1] << 1) | (s8[2] << 2) | (s8[3] << 3); ++ float dl = d*(2*scale+1); ++ aux32 &= 0xfefefefe; ++ aux32 ^= (aux32 >> 1); ++ for (int i = 0; i < 4; ++i) { ++ auto val = (const int8_t *)(iq2xxs_grid + x[ibl].qs[16*ib+4*k+i]); ++ for (int j = 0; j < 8; ++j) y4[k][QK_K*ibl+32*ib+8*i+j] = dl * val[j] * (aux32 & (1 << j) ? -1 : 1); ++ aux32 >>= 8; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq2_xxs_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_XXS_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq2_xs_r4 ++// ++ ++void quantize_row_iq2_xs_r4_ref(const float * x, block_iq2_xs_r4 * y, int64_t k) { ++ quantize_iq2_xs_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_xs_r4(const float * x, void * y, int64_t k) { ++ quantize_iq2_xs_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq2_xs(int nrows, int n_per_row, const block_iq2_xs * x, block_iq2_xs_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq2_xs * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ for (int i = 0; i < 4; ++i) { ++ uint16_t v = x4[k][ibl].qs[4*ib+i]; ++ uint8_t s = v >> 9; ++ y[ibl].qs[16*ib+4*k+i] = (v & 511) | (scrambled_sign(s) << 9); ++ } ++ y[ibl].scales[4*ib+k] = x4[k][ibl].scales[ib]; ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq2_xs_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq2_xs(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_iq2_xs, block_iq2_xs_r4, 4>(GGML_TYPE_IQ2_XS, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq2_xs); ++} ++ ++void dequantize_row_iq2_xs_r4(const block_iq2_xs_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = 0.125f*GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ float dl1 = d * (2*(x[ibl].scales[4*ib+k] & 0xf) + 1); ++ float dl2 = d * (2*(x[ibl].scales[4*ib+k] >> 4) + 1); ++ for (int i = 0; i < 4; ++i) { ++ auto val = (const int8_t *)(iq2xs_grid + (x[ibl].qs[16*ib+4*k+i] & 511)); ++ auto signs = x[ibl].qs[16*ib+4*k+i] >> 9; ++ signs ^= (signs << 1); ++ float dl = i < 2 ? dl1 : dl2; ++ for (int j = 0; j < 8; ++j) y4[k][QK_K*ibl+32*ib+8*i+j] = dl * val[j] * (signs & (1 << j) ? -1 : 1); ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq2_xs_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_XS_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq2_s_r4 ++// ++ ++void quantize_row_iq2_s_r4_ref(const float * x, block_iq2_s_r4 * y, int64_t k) { ++ quantize_iq2_s_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_s_r4(const float * x, void * y, int64_t k) { ++ quantize_iq2_s_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq2_s(int nrows, int n_per_row, const block_iq2_s * x, block_iq2_s_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq2_s * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ auto signs = x4[k][ibl].qs + QK_K/8; ++ y[ibl].d[k] = x4[k][ibl].d; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].scales[4*ib+k] = x4[k][ibl].scales[ib]; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[16*ib+4*k+i] = x4[k][ibl].qs[4*ib+i]; ++ y[ibl].signs[16*ib+4*k+i] = signs[4*ib+i]; ++ } ++ y[ibl].qh[4*ib+k] = x4[k][ibl].qh[ib]; ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq2_s_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq2_s(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_iq2_s, block_iq2_s_r4, 4>(GGML_TYPE_IQ2_S, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq2_s); ++} ++ ++void dequantize_row_iq2_s_r4(const block_iq2_s_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = 0.125f*GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ float dl1 = d * (2*(x[ibl].scales[4*ib+k] & 0xf) + 1); ++ float dl2 = d * (2*(x[ibl].scales[4*ib+k] >> 4) + 1); ++ for (int i = 0; i < 4; ++i) { ++ auto val = (const int8_t *)(iq2s_grid + (x[ibl].qs[16*ib+4*k+i] | ((x[ibl].qh[4*ib+k] << (8 - 2*i)) & 0x300))); ++ auto signs = x[ibl].signs[16*ib+4*k+i]; ++ float dl = i < 2 ? dl1 : dl2; ++ for (int j = 0; j < 8; ++j) y4[k][QK_K*ibl+32*ib+8*i+j] = dl * val[j] * (signs & (1 << j) ? -1 : 1); ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq2_s_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_S_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq3_xxs_r4 ++// ++ ++void quantize_row_iq3_xxs_r4_ref(const float * x, block_iq3_xxs_r4 * y, int64_t k) { ++ quantize_iq3_xxs_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_xxs_r4(const float * x, void * y, int64_t k) { ++ quantize_iq3_xxs_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++namespace { ++} ++ ++static void repack_iq3_xxs(int nrows, int n_per_row, const block_iq3_xxs * x, block_iq3_xxs_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq3_xxs * x4[4]; ++ uint32_t aux32; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto ysas = (uint32_t *)y[ibl].sas; ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ auto xsas = x4[k][ibl].qs + QK_K/4; ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ for (int i = 0; i < 8; ++i) { ++ y[ibl].qs[32*ib+8*k+i] = x4[k][ibl].qs[8*ib+i]; ++ } ++ std::memcpy(&aux32, xsas + 4*ib, 4); ++ uint8_t scale = aux32 >> 28; ++ uint8_t s1 = (scrambled_sign((aux32 >> 0) & 127) << 1) | ((scale >> 0) & 1); ++ uint8_t s2 = (scrambled_sign((aux32 >> 7) & 127) << 1) | ((scale >> 1) & 1); ++ uint8_t s3 = (scrambled_sign((aux32 >> 14) & 127) << 1) | ((scale >> 2) & 1); ++ uint8_t s4 = (scrambled_sign((aux32 >> 21) & 127) << 1) | ((scale >> 3) & 1); ++ aux32 = uint32_t(s1) | (uint32_t(s2) << 8) | (uint32_t(s3) << 16) | (uint32_t(s4) << 24); ++ ysas[4*ib+k] = aux32; ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq3_xxs_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq3_xxs(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<32, block_iq3_xxs, block_iq3_xxs_r4, 4>(GGML_TYPE_IQ3_XXS, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq3_xxs); ++} ++ ++void dequantize_row_iq3_xxs_r4(const block_iq3_xxs_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ uint32_t s32; ++ const uint8_t * s8 = (const uint8_t *)&s32; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const uint32_t * sas = (const uint32_t *)x[ibl].sas; ++ for (int k = 0; k < 4; ++k) { ++ const float d = 0.25f*GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ uint32_t aux32 = sas[4*ib+k]; ++ s32 = aux32 & 0x01010101; ++ uint8_t scale = s8[0] | (s8[1] << 1) | (s8[2] << 2) | (s8[3] << 3); ++ float dl = d*(2*scale+1); ++ aux32 &= 0xfefefefe; ++ aux32 ^= (aux32 >> 1); ++ for (int i = 0; i < 8; ++i) { ++ auto val = (const int8_t *)(iq3xxs_grid + x[ibl].qs[32*ib+8*k+i]); ++ for (int j = 0; j < 4; ++j) y4[k][QK_K*ibl+32*ib+4*i+j] = dl * val[j] * (aux32 & (1 << j) ? -1 : 1); ++ aux32 >>= 4; ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq3_xxs_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_XXS_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++// ++// ========================================= iq3_s_r4 ++// ++ ++void quantize_row_iq3_s_r4_ref(const float * x, block_iq3_s_r4 * y, int64_t k) { ++ quantize_iq3_s_r4(x, (void *)y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_s_r4(const float * x, void * y, int64_t k) { ++ quantize_iq3_s_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++static void repack_iq3_s(int nrows, int n_per_row, const block_iq3_s * x, block_iq3_s_r4 * y, [[maybe_unused]] bool online) { ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ int nblock = n_per_row/QK_K; ++ const block_iq3_s * x4[4]; ++ for (int row = 0; row < nrows; row += 4) { ++ for (int k = 0; k < 4; ++k) x4[k] = x + nblock*k; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ std::memset(y[ibl].scales, 0, QK_K/16); ++ std::memset(y[ibl].signs, 0, QK_K/2); ++ std::memset(y[ibl].qh, 0, QK_K/8); ++ for (int k = 0; k < 4; ++k) { ++ y[ibl].d[k] = x4[k][ibl].d; ++ for (int ib = 0; ib < QK_K/64; ++ib) { ++ int j = 8*ib + k; ++ y[ibl].scales[(j+0)%16] |= ((x4[k][ibl].scales[ib] & 0xf) << 4*((j+0)/16)); ++ y[ibl].scales[(j+4)%16] |= ((x4[k][ibl].scales[ib] >> 4) << 4*((j+4)/16)); ++ } ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ y[ibl].qh[4*ib+k] = x4[k][ibl].qh[ib]; // leave ot like this? ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[32*ib+k+8*i+0] = x4[k][ibl].qs[8*ib+i+0]; ++ y[ibl].qs[32*ib+k+8*i+4] = x4[k][ibl].qs[8*ib+i+4]; ++ } ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].signs[16*ib+4*k+i] = (((x4[k][ibl].signs[4*ib+0] >> i) & 1) << 0) | (((x4[k][ibl].signs[4*ib+0] >> (4+i)) & 1) << 1) | ++ (((x4[k][ibl].signs[4*ib+1] >> i) & 1) << 2) | (((x4[k][ibl].signs[4*ib+1] >> (4+i)) & 1) << 3) | ++ (((x4[k][ibl].signs[4*ib+2] >> i) & 1) << 4) | (((x4[k][ibl].signs[4*ib+2] >> (4+i)) & 1) << 5) | ++ (((x4[k][ibl].signs[4*ib+3] >> i) & 1) << 6) | (((x4[k][ibl].signs[4*ib+3] >> (4+i)) & 1) << 7); ++ } ++ } ++ } ++ } ++ x += 4*nblock; ++ y += nblock; ++ } ++} ++ ++size_t quantize_iq3_s_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ auto q_func = [] (const float * x, void * vy, int n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ iqkbase_quantize_iq3_s(x, (char *)vy, 1, n_per_row, imatrix, user_data); ++ }; ++ return quantize_repack<16, block_iq3_s, block_iq3_s_r4, 4>(GGML_TYPE_IQ3_S, src, dst, nrows, n_per_row, imatrix, user_data, ++ q_func, repack_iq3_s); ++} ++ ++void dequantize_row_iq3_s_r4(const block_iq3_s_r4 * x, float * y, int64_t k) { ++ auto n_per_row = k/4; ++ float * y4[4] = {y, y + n_per_row, y + 2*n_per_row, y + 3*n_per_row}; ++ int nblock = n_per_row/QK_K; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ const float d = GGML_FP16_TO_FP32(x[ibl].d[k]); ++ for (int ib = 0; ib < QK_K/32; ++ib) { ++ int l = 4*ib + k; ++ float dl = d * (1 + 2*((x[ibl].scales[l%16] >> 4*(l/16)) & 0xf)); ++ for (int i = 0; i < 4; ++i) { ++ auto grid1 = (const uint8_t *)(iq3s_grid + x[ibl].qs[32*ib+k+8*i+0] + ((x[ibl].qh[4*ib+k] << (8-i)) & 0x100)); ++ auto grid2 = (const uint8_t *)(iq3s_grid + x[ibl].qs[32*ib+k+8*i+4] + ((x[ibl].qh[4*ib+k] << (4-i)) & 0x100)); ++ for (int j = 0; j < 4; ++j) { ++ y4[k][QK_K*ibl+32*ib+4*i+ 0+j] = dl * grid1[j] * (x[ibl].signs[16*ib+4*k+j] & (1 << (i+0)) ? -1 : 1); ++ y4[k][QK_K*ibl+32*ib+4*i+16+j] = dl * grid2[j] * (x[ibl].signs[16*ib+4*k+j] & (1 << (i+4)) ? -1 : 1); ++ } ++ } ++ } ++ } ++ } ++} ++ ++void vec_dot_iq3_s_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_S_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++void quantize_row_iq1_s_r4_ref(const float * x, block_iq1_s_r4 * y, int64_t k) { ++ quantize_iq1_s_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq1_s_r4(const float * x, void * y, int64_t k) { ++ quantize_iq1_s_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++size_t quantize_iq1_s_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%kBlockSize == 0); ++ int nblock = n_per_row/kBlockSize; ++ float weight[kBlockSize]; ++ int8_t L[kBlockSize]; ++ float pairs[2*kBlockSize]; ++ float sumx[kBlockSize+1], sumw[kBlockSize+1]; ++ float max[4]; ++ uint16_t index[4]; ++ int shift; ++ float invd[4]; ++ std::vector scales(4*nblock); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ1_S_R4, n_per_row); ++ char * cy = (char *)dst; ++ for (int row = 0; row < nrows; row += 4) { ++ ggml_half * dptr = (ggml_half *)cy; ++ auto y = (block_iq1_s_r4 *)(dptr + 4); ++ for (int k = 0; k < 4; ++k) max[k] = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ auto xb = src + k*n_per_row + kBlockSize*ibl; ++ float sumx2 = 0; ++ for (int j = 0; j < kBlockSize; ++j) sumx2 += xb[j]*xb[j]; ++ if (sumx2 < 1e-14f) { ++ //printf("Found block with all zeros\n"); ++ // all zero ++ int ind = 1029; // this is the grid entry with all zeros ++ scales[4*ibl+k] = 0; ++ uint16_t h = 0; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[4*i + k] = ind & 255; ++ h |= (ind >> 8) << 3*i; ++ } ++ y[ibl].qh[k] = h; ++ continue; ++ } ++ float sigma2 = 1.5f*sumx2/kBlockSize; ++ bool have_imatrix = false; ++ if (imatrix) { ++ have_imatrix = true; ++ float sumwx = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ weight[j] = imatrix[kBlockSize*ibl + j]*sqrt(sigma2 + xb[j]*xb[j]); ++ sumwx += weight[j]*std::abs(xb[j]); ++ } ++ if (sumwx < 1e-14f) { ++ printf("Found block with mismatching importance/model weights\n"); ++ // Either all weights are zero, or xb is zero where weight is not zero. ++ // In both of these cases it is better to simply ignore the imatrix ++ have_imatrix = false; ++ } ++ } ++ if (!have_imatrix) { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = sqrt(sigma2 + xb[j]*xb[j]); ++ } ++ iqkbase_iq1s_process_1block(kBlockSize, xb, weight, L, scales.data() + 4*ibl + k, index, &shift, pairs, sumx, sumw); ++ GGML_ASSERT(scales[4*ibl+k] >= 0); ++ max[k] = std::max(max[k], scales[4*ibl+k]); ++ uint16_t h = 0; ++ for (int i = 0; i < 4; ++i) { ++ GGML_ASSERT(index[i] >= 0 && index[i] < 2048); ++ y[ibl].qs[4*i + k] = index[i] & 255; ++ h |= (index[i] >> 8) << 3*i; ++ } ++ if (shift < 0) h |= 0x8000; ++ y[ibl].qh[k] = h; ++ } ++ } ++ for (int k = 0; k < 4; ++k) { ++ dptr[k] = GGML_FP32_TO_FP16(1.0625f*max[k]/15);; ++ invd[k] = max[k] ? 15/max[k] : 0.f; ++ } ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ int ls = nearest_int(0.5f*(scales[4*ibl+k]*invd[k] - 1)); ++ ls = std::max(0, std::min(7, ls)); ++ y[ibl].qh[k] |= (ls << 12); ++ } ++ } ++ cy += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq1_s_r4(const block_iq1_s_r4 * x, float * y, int64_t n) { ++ auto dptr = (const ggml_half *)x; ++ x = (const block_iq1_s_r4 *)(dptr + 4); ++ float d[4]; ++ for (int k = 0; k < 4; ++k) d[k] = GGML_FP16_TO_FP32(dptr[k]); ++ int n_per_row = n/4; ++ GGML_ASSERT(n_per_row%32 == 0); ++ int nblock = n_per_row/32; ++ float * yk[4]; ++ for (int k = 0; k < 4; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ float shift = x[ib].qh[k] & 0x8000 ? -IQ1S_DELTA : IQ1S_DELTA; ++ float dl = d[k]*(2*((x[ib].qh[k] >> 12) & 7) + 1); ++ for (int i = 0; i < 4; ++i) { ++ auto idx = x[ib].qs[4*i+k] | (((x[ib].qh[k] >> 3*i) & 7) << 8); ++ auto grid = (const int8_t *)(iq1s_grid + idx); ++ for (int j = 0; j < 8; ++j) yk[k][32*ib + 8*i + j] = dl*(grid[j] + shift); ++ } ++ } ++ } ++} ++ ++void vec_dot_iq1_s_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ1_S_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++void quantize_row_iq1_m_r4_ref(const float * x, block_iq1_m_r4 * y, int64_t k) { ++ quantize_iq1_m_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++void quantize_row_iq1_m_r4(const float * x, void * y, int64_t k) { ++ quantize_iq1_m_r4(x, y, 4, k/4, nullptr, nullptr); ++} ++ ++size_t quantize_iq1_m_r4(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ constexpr int kBlockSize = 32; ++ GGML_ASSERT(nrows%4 == 0); ++ GGML_ASSERT(n_per_row%kBlockSize == 0); ++ int nblock = n_per_row/kBlockSize; ++ float weight[kBlockSize]; ++ int8_t L[kBlockSize]; ++ float pairs[2*kBlockSize]; ++ float max[4]; ++ uint16_t index[4]; ++ int shift1, shift2; ++ float invd[4]; ++ const uint8_t masks[4] = {0x00, 0x80, 0x08, 0x88}; ++ std::vector scales(8*nblock); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ1_M_R4, n_per_row); ++ char * cy = (char *)dst; ++ for (int row = 0; row < nrows; row += 4) { ++ ggml_half * dptr = (ggml_half *)cy; ++ auto y = (block_iq1_m_r4 *)(dptr + 4); ++ for (int k = 0; k < 4; ++k) max[k] = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ auto xb = src + k*n_per_row + kBlockSize*ibl; ++ float sumx2l = 0, sumx2h = 0; ++ for (int j = 0; j < kBlockSize/2; ++j) sumx2l += xb[j]*xb[j]; ++ for (int j = kBlockSize/2; j < kBlockSize; ++j) sumx2h += xb[j]*xb[j]; ++ float sumx2 = sumx2l + sumx2h; ++ if (sumx2 < 1e-14f) { ++ scales[8*ibl+2*k+0] = scales[8*ibl+2*k+1] = 0; ++ int ind = 1029; ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[4*i + k] = ind & 255; ++ } ++ for (int i = 0; i < 2; ++i) { ++ y[ibl].qh[4*i+k] = (ind >> 8) | ((ind >> 8) << 4); ++ } ++ continue; ++ } ++ float sigma2 = 1.5f*sumx2/kBlockSize; ++ if (imatrix) { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = imatrix[kBlockSize*ibl + j]*sqrt(sigma2 + xb[j]*xb[j]); ++ float sumwx = 0; ++ for (int j = 0; j < kBlockSize/2; ++j) sumwx += weight[j]*std::abs(xb[j]); ++ if (sumwx < 1e-14f) { ++ for (int j = 0; j < kBlockSize/2; ++j) weight[j] = sqrt(sigma2 + xb[j]*xb[j]); ++ } ++ sumwx = 0; ++ for (int j = kBlockSize/2; j < kBlockSize; ++j) sumwx += weight[j]*std::abs(xb[j]); ++ if (sumwx < 1e-14) { ++ for (int j = kBlockSize/2; j < kBlockSize; ++j) weight[j] = sqrt(sigma2 + xb[j]*xb[j]); ++ } ++ } else { ++ for (int j = 0; j < kBlockSize; ++j) weight[j] = sqrt(sigma2 + xb[j]*xb[j]); ++ } ++ if (sumx2l > 1e-14f) { ++ iqkbase_iq1m_process_1block(xb+ 0, weight+ 0, L, scales.data() + 8*ibl + 2*k+0, index+0, &shift1, pairs); ++ } else { ++ scales[8*ibl+2*k+0] = 0; ++ index[0] = index[1] = 1029; ++ } ++ if (sumx2h > 1e-14f) { ++ iqkbase_iq1m_process_1block(xb+16, weight+16, L, scales.data() + 8*ibl + 2*k+1, index+2, &shift2, pairs); ++ } else { ++ scales[8*ibl+2*k+1] = 0; ++ index[2] = index[3] = 1029; ++ } ++ max[k] = std::max(max[k], std::max(scales[8*ibl+2*k+0], scales[8*ibl+2*k+1])); ++ for (int i = 0; i < 4; ++i) { ++ y[ibl].qs[4*i + k] = index[i] & 255; ++ } ++ for (int i = 0; i < 2; ++i) { ++ y[ibl].qh[4*i+k] = (index[2*i+0] >> 8) | ((index[2*i+1] >> 8) << 4); ++ } ++ y[ibl].qh[0+k] |= masks[shift1]; ++ y[ibl].qh[4+k] |= masks[shift2]; ++ } ++ } ++ for (int k = 0; k < 4; ++k) { ++ dptr[k] = GGML_FP32_TO_FP16(1.0625f*max[k]/15);; ++ invd[k] = max[k] ? 15/max[k] : 0.f; ++ } ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ for (int k = 0; k < 4; ++k) { ++ int ls1 = nearest_int(scales[8*ibl+2*k+0]*invd[k]); ++ int ls2 = nearest_int(scales[8*ibl+2*k+1]*invd[k]); ++ ls1 = std::max(0, std::min(15, ls1)); ++ ls2 = std::max(0, std::min(15, ls2)); ++ y[ibl].scales[k] = ls1 | (ls2 << 4); ++ } ++ } ++ cy += 4*row_size; ++ src += 4*n_per_row; ++ } ++ return nrows*row_size; ++} ++ ++void dequantize_row_iq1_m_r4(const block_iq1_m_r4 * x, float * y, int64_t n) { ++ auto dptr = (const ggml_half *)x; ++ x = (const block_iq1_m_r4 *)(dptr + 4); ++ float d[4]; ++ for (int k = 0; k < 4; ++k) d[k] = GGML_FP16_TO_FP32(dptr[k]); ++ int n_per_row = n/4; ++ GGML_ASSERT(n_per_row%32 == 0); ++ int nblock = n_per_row/32; ++ float dl[2]; ++ float * yk[4]; ++ for (int k = 0; k < 4; ++k) yk[k] = y + k*n_per_row; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int k = 0; k < 4; ++k) { ++ dl[0] = d[k]*(x[ib].scales[k] & 0xf); ++ dl[1] = d[k]*(x[ib].scales[k] >> 4); ++ for (int i = 0; i < 2; ++i) { ++ auto idx1 = x[ib].qs[8*i+k+0] | ((x[ib].qh[4*i+k] & 0x07) << 8); ++ auto idx2 = x[ib].qs[8*i+k+4] | ((x[ib].qh[4*i+k] & 0x70) << 4); ++ auto grid1 = (const int8_t *)(iq1s_grid + idx1); ++ auto grid2 = (const int8_t *)(iq1s_grid + idx2); ++ auto delta1 = x[ib].qh[4*i+k] & 0x08 ? -IQ1M_DELTA : IQ1M_DELTA; ++ auto delta2 = x[ib].qh[4*i+k] & 0x80 ? -IQ1M_DELTA : IQ1M_DELTA; ++ for (int j = 0; j < 8; ++j) yk[k][32*ib + 16*i + j + 0] = dl[i]*(grid1[j] + delta1); ++ for (int j = 0; j < 8; ++j) yk[k][32*ib + 16*i + j + 8] = dl[i]*(grid2[j] + delta2); ++ } ++ } ++ } ++} ++ ++void vec_dot_iq1_m_r4_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ1_M_R4, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++void quantize_row_q8_KV(const float * x, void * vy, int64_t k) { ++ iqk_quantize_row_q8_KV(x, vy, k); ++} ++ ++void quantize_row_q8_KV_ref(const float * x, void * y, int64_t k) { ++ quantize_row_q8_KV(x, y, k); ++} ++ ++size_t quantize_q8_KV(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ (void)imatrix; ++ auto row_size = ggml_row_size(GGML_TYPE_Q8_KV, n_per_row); ++ auto q = (char *)dst; ++ for (int row = 0; row < nrows; ++row) { ++ quantize_row_q8_KV(src, q, n_per_row); ++ src += n_per_row; ++ q += row_size; ++ } ++ return row_size*nrows; ++} ++ ++void dequantize_row_q8_KV(const void * x, float * y, int64_t k) { ++ auto dptr = (const float *)x; ++ float d = dptr[0]; ++ auto q8 = (const int8_t *)(dptr + 2); ++ for (int j = 0; j < k; ++j) y[j] = d * q8[j]; ++} ++ ++void vec_dot_q8_KV_q8_KV(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_Q8_KV, vx, 0, GGML_TYPE_Q8_KV, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ GGML_ASSERT(n%QK4_NL == 0); ++ GGML_ASSERT(nrc == 1); ++ GGML_UNUSED(bs); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++} ++ ++ ++//================================================ ++ ++namespace { ++struct Repack { ++ using repack_func = void (*) (int nrows, int n_per_row, const char * src, char * dst, bool online); ++ ggml_type new_type; ++ int num_rows; ++ repack_func repack; ++}; ++struct Modify { ++ using modify_func_t = void (*)(int64_t k, char * src_dst); ++ modify_func_t mod_func; ++ int nrows; ++}; ++const Modify * get_modify_info(ggml_type type) { ++ static const std::unordered_map k_mod_map = { ++#ifdef __ARM_NEON ++ { GGML_TYPE_Q4_0_R8, {modify_q4_0_r8, 8} }, ++#endif ++#ifdef HAVE_FANCY_SIMD ++ { GGML_TYPE_Q8_0_R8, {modify_q8_0_r8, 8} }, ++ { GGML_TYPE_Q8_K_R8, {modify_q8_k_r8, 8} }, ++ { GGML_TYPE_Q8_KV_R8, {modify_q8_KV_r8, 8} }, ++#endif ++ }; ++ auto it = k_mod_map.find(type); ++ return it != k_mod_map.end() ? &it->second : nullptr; ++} ++bool is_forbidden_tensor(const std::string& name) { ++ static const std::string kTokenEmbd{"token_embd.weight"}; ++ if (name == kTokenEmbd) return true; ++ //if (auto pos = name.find("attn_kv_b.weight"); pos != std::string::npos) return true; ++ return false; ++} ++} ++ ++bool iqk_should_modify_tensor([[maybe_unused]] const struct ggml_tensor * tensor) { ++ return false; ++ //if (is_forbidden_tensor(tensor->name)) return false; ++ //auto mptr = get_modify_info(tensor->type); ++ //return mptr ? true : false; ++} ++ ++bool iqk_modify_tensor(struct ggml_tensor * tensor) { ++ return false; ++ auto mptr = get_modify_info(tensor->type); ++ if (!mptr) return false; ++ if (is_forbidden_tensor(std::string{tensor->name})) return false; ++ ++ auto& m = *mptr; ++ int nrows = ggml_nrows(tensor); ++ int nchunks = nrows/m.nrows; ++ int max_thread = std::max(1, int(std::thread::hardware_concurrency()/2)); ++ int nthread = std::min(nchunks, max_thread); ++ auto row_size = ggml_row_size(tensor->type, tensor->ne[0]); ++ std::atomic counter(0); ++ auto compute = [&counter, &m, tensor, row_size, nchunks] () { ++ int64_t n_per_call = m.nrows*tensor->ne[0]; ++ while (true) { ++ int row = counter.fetch_add(1); ++ if (row >= nchunks) break; ++ m.mod_func(n_per_call, (char *)tensor->data + row_size*row*m.nrows); ++ } ++ }; ++ std::vector workers(nthread-1); ++ for (auto& w : workers) w = std::thread(compute); ++ compute(); ++ for (auto& w : workers) w.join(); ++ ++ return true; ++} ++ ++namespace { ++const Repack * get_repack_info(ggml_type type) { ++ static const std::unordered_map k_map = { ++ { GGML_TYPE_IQ2_K, { GGML_TYPE_IQ2_K_R4, 4, (Repack::repack_func)repack_iq2_k} }, ++ { GGML_TYPE_IQ3_K, { GGML_TYPE_IQ3_K_R4, 4, (Repack::repack_func)repack_iq3_k} }, ++ { GGML_TYPE_IQ4_K, { GGML_TYPE_IQ4_K_R4, 4, (Repack::repack_func)repack_iq4_k} }, ++ { GGML_TYPE_IQ5_K, { GGML_TYPE_IQ5_K_R4, 4, (Repack::repack_func)repack_iq5_k} }, ++ { GGML_TYPE_IQ4_XS, { GGML_TYPE_IQ4_XS_R8, 8, (Repack::repack_func)repack_iq4_xs} }, ++ { GGML_TYPE_IQ4_KS, { GGML_TYPE_IQ4_KS_R4, 4, (Repack::repack_func)repack_iq4_ks} }, ++ { GGML_TYPE_IQ5_KS, { GGML_TYPE_IQ5_KS_R4, 4, (Repack::repack_func)repack_iq5_ks} }, ++ { GGML_TYPE_IQ4_NL, { GGML_TYPE_IQ4_NL_R4, 4, (Repack::repack_func)repack_iq4_nl} }, ++ { GGML_TYPE_IQ2_BN, { GGML_TYPE_IQ2_BN_R4, 4, (Repack::repack_func)repack_iq2_bn} }, ++ { GGML_TYPE_IQ2_XXS,{ GGML_TYPE_IQ2_XXS_R4,4, (Repack::repack_func)repack_iq2_xxs} }, ++ { GGML_TYPE_IQ2_XS, { GGML_TYPE_IQ2_XS_R4, 4, (Repack::repack_func)repack_iq2_xs} }, ++ { GGML_TYPE_IQ2_S, { GGML_TYPE_IQ2_S_R4, 4, (Repack::repack_func)repack_iq2_s} }, ++ { GGML_TYPE_IQ3_XXS,{ GGML_TYPE_IQ3_XXS_R4,4, (Repack::repack_func)repack_iq3_xxs} }, ++ { GGML_TYPE_IQ3_S, { GGML_TYPE_IQ3_S_R4, 4, (Repack::repack_func)repack_iq3_s} }, ++ { GGML_TYPE_Q2_K, { GGML_TYPE_Q2_K_R4, 4, (Repack::repack_func)repack_q2_k} }, ++ { GGML_TYPE_Q3_K, { GGML_TYPE_Q3_K_R4, 4, (Repack::repack_func)repack_q3_k} }, ++ { GGML_TYPE_Q4_K, { GGML_TYPE_Q4_K_R4, 4, (Repack::repack_func)repack_q4_k} }, ++ { GGML_TYPE_Q5_K, { GGML_TYPE_Q5_K_R4, 4, (Repack::repack_func)repack_q5_k} }, ++ { GGML_TYPE_Q6_K, { GGML_TYPE_Q6_K_R4, 4, (Repack::repack_func)repack_q6_k} }, ++ { GGML_TYPE_Q4_0, { GGML_TYPE_Q4_0_R8, 8, (Repack::repack_func)repack_q4_0} }, ++ { GGML_TYPE_Q5_0, { GGML_TYPE_Q5_0_R4, 4, (Repack::repack_func)repack_q5_0} }, ++ { GGML_TYPE_Q6_0, { GGML_TYPE_Q6_0_R4, 4, (Repack::repack_func)repack_q6_0} }, ++ { GGML_TYPE_Q8_0, { GGML_TYPE_Q8_0_R8, 8, (Repack::repack_func)repack_q8_0} }, ++ { GGML_TYPE_Q8_K, { GGML_TYPE_Q8_K_R8, 8, (Repack::repack_func)repack_q8_k} }, ++ { GGML_TYPE_Q8_KV, { GGML_TYPE_Q8_KV_R8, 8, (Repack::repack_func)repack_q8_KV} }, ++#ifdef __AVX512BF16__ ++ { GGML_TYPE_BF16, { GGML_TYPE_BF16_R16, 16, (Repack::repack_func)repack_bf16}}, ++ { GGML_TYPE_F16, { GGML_TYPE_BF16_R16, 16, (Repack::repack_func)repack_bf16} }, ++#endif ++ }; ++ auto it = k_map.find(type); ++ return it != k_map.end() ? &it->second : nullptr; ++} ++} ++ ++int iqk_repacked_type(const struct ggml_tensor * tensor) { ++ if (!ggml_is_contiguous(tensor)) return (int)tensor->type; ++ if (is_forbidden_tensor(tensor->name)) return (int)tensor->type; ++ auto rptr = get_repack_info(tensor->type); ++ return rptr && tensor->ne[1] % rptr->num_rows == 0 ? (int)rptr->new_type : (int)tensor->type; ++} ++ ++void iqk_repack_tensor(struct ggml_tensor * tensor) { ++ constexpr int kChunk = 8; ++ if (!tensor) return; ++ if (!ggml_is_contiguous(tensor)) return; ++ if (is_forbidden_tensor(tensor->name)) return; ++ if (tensor->ne[1] % 4) return; ++ ++ auto rptr = get_repack_info(tensor->type); ++ if (!rptr) return; ++ if (tensor->ne[1] % rptr->num_rows) return; ++ ++ auto& r = *rptr; ++ ++ auto nrows = ggml_nrows(tensor); ++ ++ int max_thread = std::max(1, int(std::thread::hardware_concurrency()/2)); ++ int num_chunks = (nrows + kChunk*r.num_rows - 1)/(kChunk*r.num_rows); ++ int nthread = std::min(num_chunks, max_thread); ++ ++ //printf("%s(%s): %s -> %s. %d rows, %d chunks, %d threads\n", __func__, tensor->name, ggml_type_name(tensor->type), ggml_type_name(r.new_type), ++ // int(tensor->ne[1]), num_chunks, nthread); ++ ++ std::atomic counter(0);; ++ auto compute = [&counter, &r, tensor, num_chunks, chunkSize = kChunk] () { ++ int nrows = ggml_nrows(tensor); ++ int n_per_row = tensor->ne[0]; ++ auto row_size = ggml_row_size(tensor->type, n_per_row); ++ std::vector qtmp(r.num_rows*row_size); ++ auto data = (char *)tensor->data; ++ while (true) { ++ int chunk = counter.fetch_add(1); ++ if (chunk >= num_chunks) break; ++ int first_row = chunk*chunkSize*r.num_rows; ++ int last_row = std::min(first_row + chunkSize*r.num_rows, nrows); ++ for (int row = first_row; row < last_row; row += r.num_rows) { ++ std::memcpy(qtmp.data(), data + row*row_size, r.num_rows*row_size); ++ //r.repack(r.num_rows, n_per_row, qtmp.data(), data + row*row_size, true); ++ r.repack(r.num_rows, n_per_row, qtmp.data(), data + row*row_size, false); ++ } ++ } ++ }; ++ std::vector workers(nthread-1); ++ for (auto& w : workers) w = std::thread(compute); ++ compute(); ++ for (auto& w : workers) w.join(); ++ ++ tensor->type = r.new_type; ++} ++ ++void dequantize_row_ms_i2s(const void * vx, float * y, int64_t k) { ++ constexpr int kBlockSize = 128; ++ constexpr int kGroupSize = kBlockSize/4; ++ GGML_ASSERT(k % kBlockSize == 0); ++ const uint8_t * x = (const uint8_t *)vx; ++ const float * dptr = (const float *)(x + k/4); ++ const float d = dptr[0]; ++ int nb = k/kBlockSize; ++ for (int ib = 0; ib < nb; ++ib) { ++ for (int ig = 0; ig < kBlockSize/kGroupSize; ++ig) { ++ int shift = 6 - 2*ig; ++ for (int j = 0; j < kGroupSize; ++j) { ++ y[j] = d * (((x[j] >> shift) & 3) - 1); ++ } ++ y += kGroupSize; ++ } ++ x += kGroupSize; ++ } ++} ++ ++namespace { ++template ++class QuantizerIQKT { ++ static_assert(group_size == 8 || group_size == 4); ++ static_assert(block_size >= 8 && block_size%8 == 0); ++public: ++ constexpr static int kSuperBlockSize = QK_K; ++ constexpr static int kBlockSize = block_size; ++ constexpr static int kGroupSize = group_size; ++ constexpr static int kNg = kBlockSize/kGroupSize; ++ constexpr static int kNblock = kSuperBlockSize/kBlockSize; ++ constexpr static int kNumVal = 1 << num_bits; // i.e, 16 bits per group of 8 ++ constexpr static float kScale = is_int ? 1.f : 31.75f; ++ constexpr static bool kVerbose = false; ++ ++ QuantizerIQKT(int num_clusters, int num_neighbours, int offset = 4096); ++ const float * values() const { return m_values.data(); } ++ ++ inline void find_best_match(float d, const float * xb, const float * weight, int * best_idx) const; ++ inline std::pair find_best_scale(const float * xb, const float * weight, const int * best_idx) const; ++ inline float find_best_inverse_scale(const float * xb, const float * weight, const int * best_idx) const; ++ ++ static inline void set_values(uint32_t i, float * result, float scale, int offset = 4096) { ++ uint32_t x = i + offset; ++ if constexpr (is_int) { ++ constexpr uint32_t ka = 0xCBAC1FED; ++ uint32_t s; ++ auto i8 = (const int8_t *)&s; ++ for (int k = 0; k < kGroupSize; ++k) { ++ x = ka*x; ++ s = x & 0x3f3f3f3f; ++ if constexpr (is_abs) { ++ result[k] = scale*std::abs(i8[0] + i8[1] + i8[2] + i8[3] - 126.f); ++ } else { ++ result[k] = scale*(i8[0] + i8[1] + i8[2] + i8[3] - 126.f); ++ } ++ } ++ } else { ++ constexpr uint32_t ka = 89226354; ++ constexpr uint32_t kb = 64248484; ++ constexpr uint32_t kmask = 0x8fff8fff; ++ constexpr uint32_t km32 = 0x3b603b60; ++ for (int k = 0; k < kGroupSize; ++k) { ++ x = ka*x + kb; ++ uint32_t s = (x & kmask) ^ km32; ++ float val = GGML_FP16_TO_FP32(s & 65535) + GGML_FP16_TO_FP32(s >> 16); ++ if constexpr (is_abs) result[k] = scale*std::abs(val); ++ else result[k] = scale*val; ++ } ++ } ++ } ++ ++ static inline int bin4(float x) { ++ if constexpr (is_abs) { ++ return x < 16.f ? 0 : x < 32.f ? 1 : x < 64.f ? 2 : 3; ++ } else { ++ return x < -24.f ? 0 : x < 0.0f ? 1 : x < 24.f ? 2 : 3; ++ } ++ } ++ static inline int bin5(float x) { ++ if constexpr (is_abs) { ++ return x < 11.2f ? 0 : x < 24.f ? 1 : x < 39.f ? 2 : x < 58.f ? 3 : 4; ++ } else { ++ return x < -48.f ? 0 : x < -16.f ? 1 : x < 16.f ? 2 : x < 48.f ? 3 : 4; ++ } ++ } ++ inline int bin3(int idim, float x) const { return x < m_mid[2*idim+0] ? 0 : x < m_mid[2*idim+1] ? 1 : 2; } ++ ++ static inline void set_weights(float sigma2_scale, int nblock, const float * x, const float * imatrix, float * row_weights) { ++ constexpr float kEps2 = 1e-14f; ++ constexpr float kWeight = 1e-4f; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ const float * xbl = x + ibl*kSuperBlockSize; ++ float * wbl = row_weights + ibl*kSuperBlockSize; ++ ++ float sumx2 = 0; ++ for (int j = 0; j < kSuperBlockSize; ++j) sumx2 += xbl[j]*xbl[j]; ++ if (sumx2 < kEps2*kSuperBlockSize) { ++ // all x in th super block are (almost) zero ++ for (int j = 0; j < kSuperBlockSize; ++j) wbl[j] = kWeight; ++ continue; ++ } ++ const float sigma2 = sigma2_scale*sumx2/kSuperBlockSize; ++ ++ if (imatrix) { ++ for (int ib = 0; ib < kSuperBlockSize/kBlockSize; ++ib) { ++ const float * qw = imatrix + ibl*kSuperBlockSize + ib*kBlockSize; ++ const float * xb = xbl + ib*kBlockSize; ++ float * wb = wbl + ib*kBlockSize; ++ float sumwx = 0, sumw2 = 0, sumx2 = 0; ++ for (int j = 0; j < kBlockSize; ++j) { ++ wb[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ sumwx += wb[j]*std::abs(xb[j]); ++ sumw2 += wb[j]*wb[j]; ++ sumx2 += xb[j]*xb[j]; ++ } ++ if (sumx2 < kEps2 || sumw2 < kEps2 || sumwx < kEps2) { ++ for (int j = 0; j < kBlockSize; ++j) wb[j] = kWeight; ++ } ++ } ++ } else { ++ for (int j = 0; j < kSuperBlockSize; ++j) wbl[j] = 0.25f*sigma2 + xbl[j]*xbl[j]; ++ } ++ } ++ } ++private: ++ static std::vector cluster_points(const std::vector& points, int ncluster, int niter, float * mid); ++ static std::vector> finalize_clusters(int num_neighbours, const std::vector& points, const std::vector& clusters, ++ std::vector>& c_values); ++ std::vector m_values; ++ std::vector m_clusters; ++ std::vector> m_in_cluster; ++ std::vector> m_c_values; ++ float m_mid[4*kGroupSize]; ++}; ++ ++template ++QuantizerIQKT::QuantizerIQKT(int num_clusters, int num_neighbours, int offset) { ++ m_values.resize(kNumVal*kGroupSize); ++ float * data = m_values.data(); ++ for (int i = 0; i < kNumVal; ++i) { ++ set_values(i, data, kScale, offset); ++ data += kGroupSize; ++ } ++ if (num_clusters == 0) return; ++ // Make 128 clusters. ++ // Note: we get a slightly better result by using 64 clusters ++ // at the expense of almost doubling the quantization time. ++ m_clusters = cluster_points(m_values, num_clusters, 200, m_mid); ++ GGML_ASSERT(!m_clusters.empty()); ++ m_in_cluster = finalize_clusters(num_neighbours, m_values, m_clusters, m_c_values); ++} ++ ++template ++std::pair QuantizerIQKT::find_best_scale( ++ const float * xb, const float * weight, const int * best_idx) const { ++ float sumqx = 0, sumq2 = 0; ++#ifdef __AVX2__ ++ auto vqx = _mm256_setzero_ps(); ++ auto vq2 = _mm256_setzero_ps(); ++ for (int l = 0; l < kBlockSize; l += 8) { ++ auto vx = _mm256_loadu_ps(xb+l); ++ auto vw = _mm256_loadu_ps(weight+l); ++ auto vq = kGroupSize == 8 ? _mm256_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize]) : ++ _mm256_set_m128(_mm_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize+1]), ++ _mm_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize+0])); ++ auto vqw = _mm256_mul_ps(vq, vw); ++ vqx = _mm256_fmadd_ps(vqw, vx, vqx); ++ vq2 = _mm256_fmadd_ps(vqw, vq, vq2); ++ } ++ sumqx = hsum_float_8(vqx); ++ sumq2 = hsum_float_8(vq2); ++#else ++ for (int l = 0; l < kNg; ++l) { ++ auto xl = xb + kGroupSize*l; ++ auto wl = weight + kGroupSize*l; ++ auto ql = m_values.data() + kGroupSize*best_idx[l]; ++ for (int k = 0; k < kGroupSize; ++k) { ++ sumqx += wl[k]*ql[k]*xl[k]; ++ sumq2 += wl[k]*ql[k]*ql[k]; ++ } ++ } ++#endif ++ return sumq2 > 0 ? std::make_pair(sumqx/sumq2, sumqx*sumqx/sumq2) : std::make_pair(0.f, 0.f); ++} ++ ++template ++float QuantizerIQKT::find_best_inverse_scale( ++ const float * xb, const float * weight, const int * best_idx) const { ++ float sumqx = 0, sumx2 = 0; ++#ifdef __AVX2__ ++ auto vqx = _mm256_setzero_ps(); ++ auto vx2 = _mm256_setzero_ps(); ++ for (int l = 0; l < kBlockSize; l += 8) { ++ auto vx = _mm256_loadu_ps(xb+l); ++ auto vw = _mm256_loadu_ps(weight+l); ++ auto vq = kGroupSize == 8 ? _mm256_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize]) : ++ _mm256_set_m128(_mm_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize+1]), ++ _mm_loadu_ps(m_values.data() + kGroupSize*best_idx[l/kGroupSize+0])); ++ auto vxw = _mm256_mul_ps(vx, vw); ++ vx2 = _mm256_fmadd_ps(vxw, vx, vx2); ++ vqx = _mm256_fmadd_ps(vxw, vq, vqx); ++ } ++ sumqx = hsum_float_8(vqx); ++ sumx2 = hsum_float_8(vx2); ++#else ++ for (int l = 0; l < kNg; ++l) { ++ auto xl = xb + kGroupSize*l; ++ auto wl = weight + kGroupSize*l; ++ auto ql = m_values.data() + kGroupSize*best_idx[l]; ++ for (int k = 0; k < kGroupSize; ++k) { ++ sumqx += wl[k]*ql[k]*xl[k]; ++ sumx2 += wl[k]*xl[k]*xl[k]; ++ } ++ } ++#endif ++ return sumx2 > 0 ? sumqx/sumx2 : 0.f; ++} ++ ++template ++void QuantizerIQKT::find_best_match(float d, ++ [[maybe_unused]] const float * xb, [[maybe_unused]] const float * weight, int * best_idx) const { ++ if (!d) { ++ std::memset(best_idx, 0, kNg*sizeof(int)); ++ return; ++ } ++ [[maybe_unused]] int ncluster = m_clusters.size()/kGroupSize; ++ [[maybe_unused]] float id = 1/d; ++#ifdef __AVX2__ ++ if constexpr (kGroupSize == 8) { ++ __m256 sqx[8]; ++ const __m256i add_idx = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); ++ float sx[8]; ++ int index[8]; ++ auto vid = _mm256_set1_ps(id); ++ auto add8 = _mm256_set1_epi32(8); ++ for (int l = 0; l < kNg; ++l) { ++ auto xl = xb + 8*l; ++ auto wl = weight + 8*l; ++ auto vx = _mm256_mul_ps(vid, _mm256_loadu_ps(xl)); ++ auto vw = _mm256_loadu_ps(wl); ++ int jbest = -1; ++ if (kGroupSize == 8 && (ncluster == 256 || ncluster == 6561)) { ++ _mm256_store_ps(sx, vx); ++ uint16_t u = 0; ++ if (ncluster == 256) { ++ for (int j = 0; j < 8; ++j) if (sx[j] > m_mid[j]) u |= (1 << j); ++ } else { ++ int s = 1; ++ for (int j = 0; j < 8; ++j) { u += s*bin3(j, sx[j]); s *= 3; } ++ } ++ jbest = u; ++ } else { ++ auto vbest = _mm256_set1_ps(INFINITY); ++ auto best_index = _mm256_set1_epi32(-1); ++ float best = INFINITY; ++ auto idx = add_idx; ++ for (int j = 0; j < ncluster; j += 8) { ++ for (int i = 0; i < 8; ++i) { ++ auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+i)); ++ auto vdiff = _mm256_sub_ps(vq, vx); ++ sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff)); ++ } ++ auto score = hsum_float_8x8(sqx); ++ auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ); ++ best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx), ++ _mm256_andnot_si256(_mm256_castps_si256(mask), best_index)); ++ vbest = _mm256_min_ps(vbest, score); ++ idx = _mm256_add_epi32(idx, add8); ++ } ++ _mm256_store_ps(sx, vbest); ++ _mm256_store_si256((__m256i *)index, best_index); ++ for (int i = 0; i < 8; ++i) { ++ if (sx[i] < best) { best = sx[i]; jbest = index[i]; } ++ } ++ } ++ auto& points = m_in_cluster[jbest]; ++ auto& values = points.empty() ? m_values : m_c_values[jbest]; ++ int npoint = values.size()/kGroupSize; ++ GGML_ASSERT(npoint > 0 && npoint%8 == 0); ++ int jbest_cluster = jbest; ++ auto vbest = _mm256_set1_ps(INFINITY); ++ auto best_index = _mm256_set1_epi32(-1); ++ auto best = INFINITY; jbest = -1; ++ auto idx = add_idx; ++ for (int j = 0; j < npoint; j += 8) { ++ for (int i = 0; i < 8; ++i) { ++ auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+i)); ++ auto vdiff = _mm256_sub_ps(vq, vx); ++ sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff)); ++ } ++ auto score = hsum_float_8x8(sqx); ++ auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ); ++ best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx), ++ _mm256_andnot_si256(_mm256_castps_si256(mask), best_index)); ++ vbest = _mm256_min_ps(vbest, score); ++ idx = _mm256_add_epi32(idx, add8); ++ } ++ _mm256_store_ps(sx, vbest); ++ _mm256_store_si256((__m256i *)index, best_index); ++ for (int i = 0; i < 8; ++i) { ++ if (sx[i] < best) { best = sx[i]; jbest = index[i]; } ++ } ++ if (jbest < 0) { ++ fprintf(stderr, "Oops: jbest = %d for cluster %d with %d points\n", jbest, jbest_cluster, int(points.size())); ++ GGML_ASSERT(false); ++ } ++ best_idx[l] = points.empty() ? jbest : points[jbest]; ++ } ++ } else { ++ __m256 sqx[4]; ++ const __m256i add_idx = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); ++ const __m256 sign_bit = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); ++ float sx[8]; ++ int index[8]; ++ auto vid_p = _mm256_set1_ps(id); ++ auto add8 = _mm256_set1_epi32(8); ++ for (int l = 0; l < kNg; ++l) { ++ auto xl = xb + 4*l; ++ auto wl = weight + 4*l; ++ auto vx4 = _mm_loadu_ps(xl); ++ auto vx = _mm256_mul_ps(vid_p, _mm256_set_m128(vx4, vx4)); ++ auto vw4 = _mm_loadu_ps(wl); ++ auto vw = _mm256_set_m128(vw4, vw4); ++ int jbest = -1; ++ if (ncluster == 256 || ncluster == 625) { ++ _mm256_storeu_ps(sx, vx); ++ uint16_t u = 0; ++ if (ncluster == 256) { ++ for (int k = 0; k < 4; ++k) u |= (bin4(sx[k]) << 2*k); ++ } else { ++ int l = 1; ++ for (int k = 0; k < 4; ++k) { u += bin5(sx[k])*l; l *= 5; } ++ } ++ jbest = u; ++ } else { ++ auto vbest = _mm256_set1_ps(INFINITY); ++ auto best_index = _mm256_set1_epi32(-1); ++ float best = INFINITY; ++ auto idx = add_idx; ++ for (int j = 0; j < ncluster; j += 8) { ++ for (int i = 0; i < 4; ++i) { ++ auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+2*i)); ++ auto vdiff = _mm256_sub_ps(vq, vx); ++ vdiff = _mm256_and_ps(sign_bit, vdiff); ++ sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, _mm256_mul_ps(vdiff, vdiff))); ++ } ++ auto score = hsum_float_4x8(sqx); ++ auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ); ++ best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx), ++ _mm256_andnot_si256(_mm256_castps_si256(mask), best_index)); ++ vbest = _mm256_min_ps(vbest, score); ++ idx = _mm256_add_epi32(idx, add8); ++ } ++ _mm256_store_ps(sx, vbest); ++ _mm256_store_si256((__m256i *)index, best_index); ++ for (int i = 0; i < 8; ++i) { ++ if (sx[i] < best) { best = sx[i]; jbest = index[i]; } ++ } ++ } ++ auto& points = m_in_cluster[jbest]; ++ auto& values = m_c_values[jbest]; ++ GGML_ASSERT(!points.empty() && points.size()%8 == 0); ++ int jbest_cluster = jbest; ++ auto vbest = _mm256_set1_ps(INFINITY); ++ auto best_index = _mm256_set1_epi32(-1); ++ float best = INFINITY; jbest = -1; ++ auto idx = add_idx; ++ for (int j = 0; j < int(points.size()); j += 8) { ++ for (int i = 0; i < 4; ++i) { ++ auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+2*i)); ++ auto vdiff = _mm256_sub_ps(vq, vx); ++ sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff)); ++ } ++ auto score = hsum_float_4x8(sqx); ++ auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ); ++ best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx), ++ _mm256_andnot_si256(_mm256_castps_si256(mask), best_index)); ++ vbest = _mm256_min_ps(vbest, score); ++ idx = _mm256_add_epi32(idx, add8); ++ } ++ _mm256_store_ps(sx, vbest); ++ _mm256_store_si256((__m256i *)index, best_index); ++ for (int i = 0; i < 8; ++i) { ++ if (sx[i] < best) { best = sx[i]; jbest = index[i]; } ++ } ++ if (jbest < 0) { ++ fprintf(stderr, "Oops: jbest = %d for cluster %d with %d points\n", jbest, jbest_cluster, int(points.size())); ++ GGML_ASSERT(false); ++ } ++ best_idx[l] = points[jbest]; ++ } ++ } ++#else ++ // TODO ++ std::memset(best_idx, 0, kNg*sizeof(int)); ++#endif ++} ++ ++template ++std::vector> QuantizerIQKT::finalize_clusters(int num_neighbours, ++ const std::vector& values, const std::vector& clusters, std::vector>& c_values) { ++ int ncluster = clusters.size()/kGroupSize; ++ std::vector> p_in_cluster(ncluster); ++ std::vector which_cluster(num_neighbours*kNumVal); ++ std::vector ibest(num_neighbours); ++ std::vector best(num_neighbours); ++ for (int ip = 0; ip < kNumVal; ++ip) { ++ auto vp = values.data() + ip*kGroupSize; ++ for (int j = 0; j < num_neighbours; ++j) { ++ best[j] = INFINITY; ibest[j] = -1; ++ } ++ for (int ic = 0; ic < ncluster; ++ic) { ++ auto vc = clusters.data() + ic*kGroupSize; ++ float dist2 = 0; ++ for (int k = 0; k < kGroupSize; ++k) { ++ float d = vp[k] - vc[k]; dist2 += d*d; ++ } ++ for (int j = 0; j < num_neighbours; ++j) { ++ if (dist2 < best[j]) { ++ for (int k = num_neighbours-1; k > j; --k) { ++ best[k] = best[k-1]; ibest[k] = ibest[k-1]; ++ } ++ best[j] = dist2; ibest[j] = ic; ++ break; ++ } ++ } ++ } ++ for (int j = 0; j < num_neighbours; ++j) { ++ if (ibest[j] < 0) { ++ printf("Oops: ibest[%d] = %d\n", j, ibest[j]); ++ } ++ GGML_ASSERT(ibest[j] >= 0); ++ p_in_cluster[ibest[j]].push_back(ip); ++ } ++ std::memcpy(which_cluster.data() + num_neighbours*ip, ibest.data(), num_neighbours*sizeof(int)); ++ } ++ std::vector> extra; ++ extra.reserve(kNumVal); ++ for (int ic = 0; ic < ncluster; ++ic) { ++ auto& points = p_in_cluster[ic]; ++ if (!points.empty() && points.size()%8 == 0) continue; ++ extra.clear(); ++ auto vc = clusters.data() + ic*kGroupSize; ++ for (int ip = 0; ip < kNumVal; ++ip) { ++ bool can_add = true; ++ for (int j = 0; j < num_neighbours; ++j) { ++ if (which_cluster[num_neighbours*ip+j] == ic) { can_add = false; break; } ++ } ++ if (!can_add) continue; ++ auto vp = values.data() + ip*kGroupSize; ++ float dist2 = 0; ++ for (int k = 0; k < kGroupSize; ++k) { ++ float d = vp[k] - vc[k]; dist2 += d*d; ++ } ++ extra.push_back(std::make_pair(dist2, ip)); ++ } ++ std::sort(extra.begin(), extra.end()); ++ int nadd = 8*((points.size()+7)/8) - points.size(); ++ for (int i = 0; i < nadd; ++i) points.push_back(extra[i].second); ++ GGML_ASSERT(points.size()%8 == 0); ++ } ++ auto min = p_in_cluster.front().size(), max = p_in_cluster.front().size(); ++ for (auto& points : p_in_cluster) { ++ min = std::min(min, points.size()); ++ max = std::max(max, points.size()); ++ } ++ c_values.resize(p_in_cluster.size()); ++ for (int i = 0; i < int(p_in_cluster.size()); ++i) { ++ auto& points = p_in_cluster[i]; ++ c_values[i].resize(points.size()*kGroupSize); ++ auto ptr = c_values[i].data(); ++ for (auto j : points) { ++ std::memcpy(ptr, values.data() + j*kGroupSize, kGroupSize*sizeof(float)); ++ ptr += kGroupSize; ++ } ++ } ++ ++ if (kVerbose) { ++ printf("%s: prepared %d clusters\n", __func__, ncluster); ++ printf(" min number of points in a cluster: %d\n", int(min)); ++ printf(" max number of points in a cluster: %d\n", int(max)); ++ } ++ return p_in_cluster; ++} ++ ++template ++std::vector QuantizerIQKT::cluster_points(const std::vector& points, int ncluster, int niter, float * mid) { ++ constexpr int ndim = kGroupSize; ++ GGML_ASSERT(points.size() % ndim == 0); ++ int npoint = points.size() / ndim; ++ GGML_ASSERT(npoint >= 2*ncluster); ++ std::vector> range(ndim, std::make_pair(INFINITY, -INFINITY)); ++ double Fo = 0; ++ for (int i = 0; i < npoint; ++i) { ++ auto v = points.data() + i*ndim; ++ for (int k = 0; k < ndim; ++k) { ++ Fo += v[k]*v[k]; ++ range[k].first = std::min(range[k].first, v[k]); ++ range[k].second = std::max(range[k].second, v[k]); ++ } ++ } ++ if (kVerbose) printf("%s (ndim = %d, npoint = %d): Fo = %g\n", __func__, ndim, npoint, Fo/points.size()); ++ if constexpr (is_abs) { ++ std::vector P(npoint); ++ for (int idim = 0; idim < ndim; ++idim) { ++ for (int ip = 0; ip < npoint; ++ip) P[ip] = points[ip*ndim+idim]; ++ std::sort(P.begin(), P.end()); ++ if (ndim == 8 && ncluster == 6561) { ++ mid[2*idim + 0] = P[npoint/3]; ++ mid[2*idim + 1] = P[2*npoint/3]; ++ } else { ++ mid[idim] = npoint%2 == 0 ? 0.5f*(P[npoint/2] + P[npoint/2-1]) : P[npoint/2]; ++ if (kVerbose) printf("%s: mid[%d] = %g\n", __func__, idim, mid[idim]); ++ } ++ } ++ } else { ++ for (int k = 0; k < ndim; ++k) mid[k] = 0.5f*(range[k].first + range[k].second); ++ } ++ std::vector sump(ncluster*ndim); ++ std::vector counts(ncluster); ++ std::vector result(ncluster*ndim); ++ if (ndim == 8 && (ncluster == 256 || ncluster == 6561)) { ++ std::memset(sump.data(), 0, sump.size()*sizeof(float)); ++ std::memset(counts.data(), 0, counts.size()*sizeof(int)); ++ for (int ip = 0; ip < npoint; ++ip) { ++ auto vp = points.data() + ndim*ip; ++ uint16_t u = 0; ++ if (ncluster == 256) { ++ for (int k = 0; k < ndim; ++k) if (vp[k] > mid[k]) u |= (1 << k); ++ } else { ++ int s = 1; ++ for (int k = 0; k < ndim; ++k) { ++ int bin = vp[k] < mid[2*k+0] ? 0 : vp[k] < mid[2*k+1] ? 1 : 2; ++ u += s*bin; s *= 3; ++ } ++ } ++ ++counts[u]; ++ for (int k = 0; k < ndim; ++k) sump[ndim*u + k] += vp[k]; ++ } ++ for (int ic = 0; ic < ncluster; ++ic) { ++ if (!counts[ic]) { ++ printf("%s: Oops. Cluster %d has no points\n", __func__, ic); ++ GGML_ABORT("fatal error"); ++ } ++ for (int k = 0; k < ndim; ++k) result[ic*ndim + k] = sump[ic*ndim + k]/counts[ic]; ++ } ++ return result; ++ } ++ else if (ndim == 4 && (ncluster == 256 || ncluster == 625)) { ++ std::memset(sump.data(), 0, sump.size()*sizeof(float)); ++ std::memset(counts.data(), 0, counts.size()*sizeof(int)); ++ for (int ip = 0; ip < npoint; ++ip) { ++ auto vp = points.data() + ndim*ip; ++ uint16_t u = 0; ++ if (ncluster == 256) { ++ for (int k = 0; k < ndim; ++k) u |= (bin4(vp[k]) << 2*k); ++ } else { ++ int s = 1; ++ for (int k = 0; k < ndim; ++k) { u += s*bin5(vp[k]); s *= 5; } ++ } ++ if (u >= int(counts.size())) { ++ printf("Oops: u = %u, vp = %g, %g, %g, %g\n", u, vp[0], vp[1], vp[2], vp[3]); ++ u = 0; ++ if (ncluster == 256) { ++ for (int k = 0; k < ndim; ++k) { ++ auto bin = bin4(vp[k]); u |= (bin << 2*k); ++ printf(" bin[%d] = %d, u = %u", k, bin, u); ++ } ++ } else { ++ for (int k = 0; k < ndim; ++k) printf(" bin[%d] = %d", k, bin5(vp[k])); ++ } ++ printf("\n"); ++ GGML_ABORT("fatal error"); ++ } ++ ++counts[u]; ++ for (int k = 0; k < ndim; ++k) sump[ndim*u + k] += vp[k]; ++ } ++ int nzero = 0; ++ for (int ic = 0; ic < ncluster; ++ic) { ++ if (!counts[ic]) { ++ ++nzero; ++ printf("%s: Oops. Cluster %d has no points: ", __func__, ic); ++ for (int k = 0; k < ndim; ++k) { ++ int l = (ic >> 2*k) & 3; ++ printf(" %d", l); ++ } ++ printf("\n"); ++ } else { ++ for (int k = 0; k < ndim; ++k) result[ic*ndim + k] = sump[ic*ndim + k]/counts[ic]; ++ } ++ } ++ if (nzero > 0) printf("%s: %d out of %d clusters dir not have any points\n", __func__, nzero, ncluster); ++ return result; ++ } ++ std::mt19937 rndm(1234); ++ float scale = 1.f/4294967296.f; ++ for (int i = 0; i < ncluster; ++i) { ++ auto v = result.data() + i*ndim; ++ for (int k = 0; k < ndim; ++k) v[k] = range[k].first + (range[k].second - range[k].first)*scale*rndm(); ++ } ++ std::vector which_cluster(npoint, -1); ++ double Flast = Fo; ++ for (int iter = 0; iter < niter; ++iter) { ++ std::memset(sump.data(), 0, sump.size()*sizeof(float)); ++ std::memset(counts.data(), 0, counts.size()*sizeof(int)); ++ int nchanged = 0; ++ double F = 0; ++ for (int ip = 0; ip < npoint; ++ip) { ++ auto vp = points.data() + ndim*ip; ++ float best = INFINITY; int ibest = -1; ++ for (int ic = 0; ic < ncluster; ++ic) { ++ auto vc = result.data() + ndim*ic; ++ float dist2 = 0; ++ for (int k = 0; k < ndim; ++k) { ++ float d = vp[k] - vc[k]; dist2 += d*d; ++ } ++ if (dist2 < best) { ++ best = dist2; ibest = ic; ++ } ++ } ++ if (ibest < 0) { ++ printf("Oops(iteration %d) - failed to find cluster for point", iter); ++ for (int k = 0; k < ndim; ++k) printf(" %g", vp[k]); ++ printf("\nHave %d clusters\n", ncluster); ++ } ++ GGML_ASSERT(ibest >= 0); ++ F += best; ++ if (which_cluster[ip] != ibest) ++nchanged; ++ which_cluster[ip] = ibest; ++ ++counts[ibest]; ++ auto vc = sump.data() + ndim*ibest; ++ for (int k = 0; k < ndim; ++k) vc[k] += vp[k]; ++ } ++ if (nchanged == 0) break; ++ for (int ic = 0; ic < ncluster; ++ic) { ++ float norm = counts[ic] > 0 ? 1.f/counts[ic] : 0.f; ++ auto vc = sump.data() + ndim*ic; ++ auto r = result.data() + ndim*ic; ++ for (int k = 0; k < ndim; ++k) r[k] = vc[k]*norm; ++ } ++ if (kVerbose) printf("%s(iteration %d): F = %g, nchanged = %d\n", __func__, iter+1, F/points.size(), nchanged); ++ if (iter > 1 && Flast/F - 1 < 1e-6) break; ++ Flast = F; ++ } ++ int nzero = 0; ++ for (int ic = 0; ic < ncluster; ++ic) { ++ if (!counts[ic]) ++nzero; ++ } ++ if (nzero > 0) printf("%s: there are %d empty clusters\n", __func__, nzero); ++ return result; ++} ++ ++// ========================================== iq1_kt ==================================================== ++ ++using QuantizerIQ1KT = QuantizerIQKT<32, 8, 13, false, true>; ++ ++const QuantizerIQ1KT& iq1kt_quantizer() { ++ static std::mutex mutex; ++ static std::unique_ptr quantizer; ++ std::lock_guard lock(mutex); ++ if (!quantizer) quantizer = std::make_unique(256, 32); ++ return *quantizer; ++} ++ ++void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights, ++ int * all_idx) { ++ ++ constexpr float kSigmaScale = 2.0f; ++ using Q = QuantizerIQ1KT; ++ ++ static_assert(Q::kNumVal%8 == 0); ++ ++ float * dptr = (float *)vy; ++ ++ block_iq1_kt * y = (block_iq1_kt *)(dptr + 1); ++ ++ int best_idx[2*Q::kNg]; ++ ++ auto& quantizer = iq1kt_quantizer(); ++ ++ int nblock = n_per_row / Q::kSuperBlockSize; ++ ++ Q::set_weights(kSigmaScale, nblock, x, quant_weights, all_weights); ++ ++ float amax_row = 0; ++ for (int j = 0; j < n_per_row; ++j) { ++ amax_row = std::max(amax_row, std::abs(x[j])); ++ } ++ ++ float amax_scale = 0, max_scale = 0; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq1_kt)); ++ ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ float amax = 0; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ float ax = std::abs(xb[j]); ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0.0f; ++ for (int ig = 0; ig < Q::kNg; ++ig) all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig] = 0; ++ continue; ++ } ++ float scale_0 = std::max(90.f, 124.f*amax/amax_row); ++ quantizer.find_best_match( amax/scale_0, xb, weight, best_idx); ++ auto [dp, score_p] = quantizer.find_best_scale(xb, weight, best_idx); ++ quantizer.find_best_match(-amax/scale_0, xb, weight, best_idx + Q::kNg); ++ auto [dm, score_m] = quantizer.find_best_scale(xb, weight, best_idx + Q::kNg); ++ ++ auto idx = best_idx; ++ if (score_p > score_m) scales[ib] = dp; ++ else { ++ scales[ib] = dm; idx += Q::kNg; score_p = score_m; ++ } ++ for (int ig = 0; ig < Q::kNg; ++ig) all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig] = idx[ig]; ++ ++ scale_0 -= 8; ++ quantizer.find_best_match( amax/scale_0, xb, weight, best_idx); ++ auto [dp1, score_p1] = quantizer.find_best_scale(xb, weight, best_idx); ++ quantizer.find_best_match(-amax/scale_0, xb, weight, best_idx + Q::kNg); ++ auto [dm1, score_m1] = quantizer.find_best_scale(xb, weight, best_idx + Q::kNg); ++ ++ if (score_p1 > score_p || score_m1 > score_p) { ++ idx = best_idx; ++ if (score_p1 > score_m1) scales[ib] = dp1; ++ else { ++ scales[ib] = dm1; idx += Q::kNg; ++ } ++ for (int ig = 0; ig < Q::kNg; ++ig) all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig] = idx[ig]; ++ } ++ ++ float abs_scale = std::abs(scales[ib]); ++ if (abs_scale > amax_scale) { ++ amax_scale = abs_scale; ++ max_scale = scales[ib]; ++ } ++ } ++ ++ } ++ ++ if (!max_scale) { ++ *dptr = 0; ++ return; ++ } ++ ++ float d = max_scale/iq4k_values[0]; ++ float best = 0; ++ for (int itry = -9; itry <= 9; ++itry) { ++ float id = (itry + iq4k_values[0])/max_scale; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const float * xb = x + ibl*Q::kSuperBlockSize; ++ const float * wb = all_weights + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ int ls = best_index_iq4nl(iq4k_values, id*scales[ib]); ++ float dl = iq4k_values[ls]; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ auto qb = quantizer.values() + Q::kGroupSize*all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig]; ++ for (int j = 0; j < Q::kGroupSize; ++j) { ++ int jj = ig*Q::kGroupSize + j; ++ float q = dl*qb[j]; ++ sumqx += wb[jj]*xb[jj]*q; ++ sumq2 += wb[jj]*q*q; ++ } ++ } ++ xb += Q::kBlockSize; ++ wb += Q::kBlockSize; ++ } ++ } ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; ++ } ++ } ++ ++ float id = d ? 1/d : 0.f; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ int ls = best_index_iq4nl(iq4k_values, id*scales[ib]); ++ y[ibl].sh[ib] = ls; ++ } ++ } ++ ++ *dptr = d; ++ if (!d) return; ++ ++ for (int iloop = 0; iloop < 1; ++iloop) { ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ int ls = iq4k_values[y[ibl].sh[ib] & 0xf]; ++ float dl = d*ls; ++ quantizer.find_best_match(dl, xb, weight, best_idx); ++ ++ auto prev_idx = all_idx + (ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize; ++ ++ float mse1 = 0, mse2 = 0; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ auto q1 = quantizer.values() + Q::kGroupSize*prev_idx[ig]; ++ auto q2 = quantizer.values() + Q::kGroupSize*best_idx[ig]; ++ for (int j = 0; j < Q::kGroupSize; ++j) { ++ int jj = ig*Q::kGroupSize + j; ++ float diff1 = xb[jj] - dl*q1[j]; ++ float diff2 = xb[jj] - dl*q2[j]; ++ mse1 += weight[jj]*diff1*diff1; ++ mse2 += weight[jj]*diff2*diff2; ++ } ++ } ++ if (mse1 < mse2) { ++ for (int ig = 0; ig < Q::kNg; ++ig) best_idx[ig] = prev_idx[ig]; ++ } else { ++ for (int ig = 0; ig < Q::kNg; ++ig) prev_idx[ig] = best_idx[ig]; ++ } ++ ++ for (int j = 0; j < Q::kNg; ++j) { ++ y[ibl].ql[ib*Q::kNg+j] = best_idx[j] & 0xff; ++ y[ibl].qh[(ib%(Q::kNblock/2))*Q::kNg+j] |= (((best_idx[j] >> 8) & 0xf) << 4*(ib/(Q::kNblock/2))); ++ y[ibl].sh[ib] |= ((best_idx[j] >> 12) << (4+j)); ++ auto xl = xb + Q::kGroupSize*j; ++ auto wl = weight + Q::kGroupSize*j; ++ auto ql = quantizer.values() + best_idx[j]*Q::kGroupSize; ++ for (int k = 0; k < Q::kGroupSize; ++k) { ++ float q = ql[k]*ls; ++ sumqx += wl[k]*xl[k]*q; ++ sumq2 += wl[k]*q*q; ++ } ++ } ++ } ++ } ++ if (sumq2 > 0) { ++ d = sumqx/sumq2; ++ *dptr = d * 1.07f; ++ if (!d) return; ++ } else { ++ break; ++ } ++ ++ } ++ ++} ++} ++ ++void quantize_row_iq1_kt_ref(const float * GGML_RESTRICT x, block_iq1_kt * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq1_kt(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq1_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq1_kt * y = (block_iq1_kt *)vy; ++ quantize_row_iq1_kt_ref(x, y, k); ++} ++ ++size_t quantize_iq1_kt(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ1_KT, n_per_row); ++ std::vector scales(n_per_row/QuantizerIQ1KT::kBlockSize); ++ std::vector weights(n_per_row); ++ std::vector idx(n_per_row/QuantizerIQ1KT::kGroupSize); ++ char * qrow = (char *)dst; ++ for (int64_t row = 0; row < nrows; ++row) { ++ quantize_row_iq1_kt_impl(src, (void *)qrow, n_per_row, imatrix, scales.data(), weights.data(), idx.data()); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq1_kt(const block_iq1_kt * x, float * y, int64_t k) { ++ assert(k % QuantizerIQ1KT::kSuperBlockSize == 0); ++ using Q = QuantizerIQ1KT; ++ const int nb = k / Q::kSuperBlockSize; ++ const float * dptr = (const float *)x; ++ const float d = *dptr * Q::kScale; ++ x = (const block_iq1_kt *)(dptr + 1); ++ auto& deq = iq1kt_quantizer(); ++ for (int ibl = 0; ibl < nb; ++ibl) { ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ float sl = d * iq4k_values[x[ibl].sh[ib] & 0xf]; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ uint16_t idx = x[ibl].ql[ib*Q::kNg + ig] | ((x[ibl].qh[(ib%(Q::kNblock/2))*Q::kNg + ig] << (8 - 4*(ib/(Q::kNblock/2)))) & 0xf00); ++ idx |= (x[ibl].sh[ib] << (8 - ig) & 0x1000); ++ deq.set_values(idx, y, sl); ++ y += Q::kGroupSize; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq1_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ1_KT, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++} ++ ++// ========================================== iq2_kt ==================================================== ++ ++namespace { ++ ++using QuantizerIQ2KT = QuantizerIQKT<32, 8, 16, false, true>; ++ ++const QuantizerIQ2KT& iq2kt_quantizer() { ++ static std::mutex mutex; ++ static std::unique_ptr quantizer; ++ std::lock_guard lock(mutex); ++ if (!quantizer) quantizer = std::make_unique(256, 8); ++ return *quantizer; ++} ++ ++void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights, ++ int * all_idx) { ++ ++ constexpr float kSigmaScale = 2.0f; ++ using Q = QuantizerIQ2KT; ++ ++ static_assert(Q::kNumVal%8 == 0); ++ ++ float * dptr = (float *)vy; ++ ++ block_iq2_kt * y = (block_iq2_kt *)(dptr + 1); ++ ++ int best_idx[2*Q::kNg]; ++ ++ auto& quantizer = iq2kt_quantizer(); ++ ++ int nblock = n_per_row / Q::kSuperBlockSize; ++ ++ Q::set_weights(kSigmaScale, nblock, x, quant_weights, all_weights); ++ ++ float amax_row = 0; ++ for (int j = 0; j < n_per_row; ++j) { ++ amax_row = std::max(amax_row, std::abs(x[j])); ++ } ++ ++ float amax_scale = 0, max_scale = 0; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq2_kt)); ++ ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ float amax = 0; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ float ax = std::abs(xb[j]); ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0.0f; ++ for (int ig = 0; ig < Q::kNg; ++ig) all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig] = 0; ++ continue; ++ } ++ float scale_0 = std::max(90.f, 124.f*amax/amax_row); ++ quantizer.find_best_match( amax/scale_0, xb, weight, best_idx); ++ auto [dp, score_p] = quantizer.find_best_scale(xb, weight, best_idx); ++ quantizer.find_best_match(-amax/scale_0, xb, weight, best_idx + Q::kNg); ++ auto [dm, score_m] = quantizer.find_best_scale(xb, weight, best_idx + Q::kNg); ++ ++ auto idx = best_idx; ++ if (score_p > score_m) scales[ib] = dp; ++ else { ++ scales[ib] = dm; idx += Q::kNg; ++ } ++ for (int ig = 0; ig < Q::kNg; ++ig) all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig] = idx[ig]; ++ ++ float abs_scale = std::abs(scales[ib]); ++ if (abs_scale > amax_scale) { ++ amax_scale = abs_scale; ++ max_scale = scales[ib]; ++ } ++ } ++ ++ } ++ ++ if (!max_scale) { ++ *dptr = 0; ++ return; ++ } ++ ++ float d = max_scale/iq4k_values[0]; ++ float best = 0; ++ for (int itry = -9; itry <= 9; ++itry) { ++ float id = (itry + iq4k_values[0])/max_scale; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const float * xb = x + ibl*Q::kSuperBlockSize; ++ const float * wb = all_weights + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ int ls = best_index_iq4nl(iq4k_values, id*scales[ib]); ++ float dl = iq4k_values[ls]; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ auto qb = quantizer.values() + Q::kGroupSize*all_idx[(ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize + ig]; ++ for (int j = 0; j < Q::kGroupSize; ++j) { ++ int jj = ig*Q::kGroupSize + j; ++ float q = dl*qb[j]; ++ sumqx += wb[jj]*xb[jj]*q; ++ sumq2 += wb[jj]*q*q; ++ } ++ } ++ xb += Q::kBlockSize; ++ wb += Q::kBlockSize; ++ } ++ } ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; ++ } ++ } ++ ++ float id = d ? 1/d : 0.f; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock/2; ++ib) { ++ int ls1 = best_index_iq4nl(iq4k_values, id*scales[ib]); ++ int ls2 = best_index_iq4nl(iq4k_values, id*scales[ib + Q::kNblock/2]); ++ y[ibl].scales[ib] = ls1 | (ls2 << 4); ++ } ++ } ++ ++ *dptr = d; ++ if (!d) return; ++ ++ for (int iloop = 0; iloop < 1; ++iloop) { ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ auto qs = (uint16_t *)y[ibl].ql; ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ int ls = iq4k_values[(y[ibl].scales[ib%(Q::kNblock/2)] >> 4*(ib/(Q::kNblock/2))) & 0xf]; ++ float dl = d*ls; ++ quantizer.find_best_match(dl, xb, weight, best_idx); ++ ++ auto prev_idx = all_idx + (ibl*Q::kSuperBlockSize + ib*Q::kBlockSize)/Q::kGroupSize; ++ ++ float mse1 = 0, mse2 = 0; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ auto q1 = quantizer.values() + Q::kGroupSize*prev_idx[ig]; ++ auto q2 = quantizer.values() + Q::kGroupSize*best_idx[ig]; ++ for (int j = 0; j < Q::kGroupSize; ++j) { ++ int jj = ig*Q::kGroupSize + j; ++ float diff1 = xb[jj] - dl*q1[j]; ++ float diff2 = xb[jj] - dl*q2[j]; ++ mse1 += weight[jj]*diff1*diff1; ++ mse2 += weight[jj]*diff2*diff2; ++ } ++ } ++ if (mse1 < mse2) { ++ for (int ig = 0; ig < Q::kNg; ++ig) best_idx[ig] = prev_idx[ig]; ++ } else { ++ for (int ig = 0; ig < Q::kNg; ++ig) prev_idx[ig] = best_idx[ig]; ++ } ++ ++ for (int j = 0; j < Q::kNg; ++j) { ++ qs[j] = best_idx[j]; ++ auto xl = xb + Q::kGroupSize*j; ++ auto wl = weight + Q::kGroupSize*j; ++ auto ql = quantizer.values() + best_idx[j]*Q::kGroupSize; ++ for (int k = 0; k < Q::kGroupSize; ++k) { ++ float q = ql[k]*ls; ++ sumqx += wl[k]*xl[k]*q; ++ sumq2 += wl[k]*q*q; ++ } ++ } ++ qs += Q::kNg; ++ } ++ } ++ if (sumq2 > 0) { ++ d = sumqx/sumq2; ++ *dptr = d; ++ if (!d) return; ++ } else { ++ break; ++ } ++ ++ if (false) { ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ auto qs = (uint16_t *)y[ibl].ql; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ for (int j = 0; j < Q::kNg; ++j) best_idx[j] = qs[ib*Q::kNg+j]; ++ auto pair = quantizer.find_best_scale(xb, weight, best_idx); ++ scales[ib] = pair.first; ++ } ++ } ++ float id = d ? 1/d : 0.f; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock/2; ++ib) { ++ int ls1 = best_index_iq4nl(iq4k_values, id*scales[ib]); ++ int ls2 = best_index_iq4nl(iq4k_values, id*scales[ib + Q::kNblock/2]); ++ y[ibl].scales[ib] = ls1 | (ls2 << 4); ++ } ++ } ++ } ++ ++ } ++ ++} ++} ++ ++void quantize_row_iq2_kt_ref(const float * GGML_RESTRICT x, block_iq2_kt * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq2_kt(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq2_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq2_kt * y = (block_iq2_kt *)vy; ++ quantize_row_iq2_kt_ref(x, y, k); ++} ++ ++size_t quantize_iq2_kt(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ2_KT, n_per_row); ++ std::vector scales(n_per_row/QuantizerIQ2KT::kBlockSize); ++ std::vector weights(n_per_row); ++ std::vector idx(n_per_row/QuantizerIQ2KT::kGroupSize); ++ char * qrow = (char *)dst; ++ for (int64_t row = 0; row < nrows; ++row) { ++ quantize_row_iq2_kt_impl(src, (void *)qrow, n_per_row, imatrix, scales.data(), weights.data(), idx.data()); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq2_kt(const block_iq2_kt * x, float * y, int64_t k) { ++ assert(k % QuantizerIQ2KT::kSuperBlockSize == 0); ++#ifdef __AVX2__ ++ //if (iqk_dequantize_ktquants(GGML_TYPE_IQ2_KT, k, x, 0, y, 0, 1)) return; ++#endif ++ const int nb = k / QuantizerIQ2KT::kSuperBlockSize; ++ const float * dptr = (const float *)x; ++ const float d = *dptr * QuantizerIQ2KT::kScale; ++ x = (const block_iq2_kt *)(dptr + 1); ++ auto& deq = iq2kt_quantizer(); ++ for (int ibl = 0; ibl < nb; ++ibl) { ++ auto yl = y + ibl*QuantizerIQ2KT::kSuperBlockSize; ++ auto yh = yl + QuantizerIQ2KT::kSuperBlockSize/2; ++ const uint16_t * ql = (const uint16_t *)x[ibl].ql; ++ const uint16_t * qh = ql + QuantizerIQ2KT::kNg*QuantizerIQ2KT::kNblock/2; ++ for (int ib = 0; ib < QuantizerIQ2KT::kNblock/2; ++ib) { ++ float sl = d * iq4k_values[x[ibl].scales[ib] & 0xf]; ++ float sh = d * iq4k_values[x[ibl].scales[ib] >> 4]; ++ for (int ig = 0; ig < QuantizerIQ2KT::kNg; ++ig) { ++ deq.set_values(ql[ig], yl, sl); ++ deq.set_values(qh[ig], yh, sh); ++ yl += QuantizerIQ2KT::kGroupSize; ++ yh += QuantizerIQ2KT::kGroupSize; ++ } ++ ql += QuantizerIQ2KT::kNg; ++ qh += QuantizerIQ2KT::kNg; ++ } ++ } ++} ++ ++void vec_dot_iq2_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_KT, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++} ++ ++namespace { ++ ++using QuantizerIQ3KT = QuantizerIQKT<32, 8, 16, true, true>; ++const QuantizerIQ3KT& iq3kt_quantizer() { ++ static std::mutex mutex; ++ std::lock_guard lock(mutex); ++ static std::unique_ptr quantizer; ++ if (!quantizer) quantizer = std::make_unique(256, 8); ++ return *quantizer; ++} ++ ++void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, ++ float * all_weights, float * qtmp) { ++ ++ constexpr float kSigmaScale = 2.0f; ++ constexpr float kStep = 8.0f; ++ ++ using Q = QuantizerIQ3KT; ++ ++ static_assert(Q::kNumVal%8 == 0); ++ ++ constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize; ++ ++ float * dptr = (float *)vy; ++ ++ block_iq3_kt * y = (block_iq3_kt *)(dptr + 1); ++ ++ int best_idx[2*Q::kNg]; ++ ++ auto& quantizer = iq3kt_quantizer(); ++ ++ int nblock = n_per_row / Q::kSuperBlockSize; ++ ++ float amax_row = 0; ++ for (int j = 0; j < n_per_row; ++j) amax_row = std::max(amax_row, std::abs(x[j])); ++ if (!amax_row) { ++ *dptr = 0.f; ++ std::memset(y, 0, nblock*sizeof(block_iq3_kt)); ++ return; ++ } ++ ++ Q::set_weights(kSigmaScale, nblock, x, quant_weights, all_weights); ++ ++ float amax_scale = 0, max_scale = 0; ++ ++ float xaux[Q::kBlockSize]; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq3_kt)); ++ ++ auto scales = all_scales + ibl*Q::kNblock; ++ auto xbl = x + ibl*Q::kSuperBlockSize; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ float amax = 0; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ float ax = std::abs(xb[j]); ++ xaux[j] = ax; ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0.0f; ++ continue; ++ } ++ ++ //quantizer.find_best_match(amax/96.f, xaux, weight, best_idx+Q::kNg); ++ //scales[ib] = quantizer.find_best_scale(xaux, weight, best_idx+Q::kNg).first; ++ ++ float scale_0 = std::max(84.f, 123.f*amax/amax_row); ++ //float scale_0 = std::max(64.f, 123.f*amax/amax_row); ++ float best = 0; ++ bool found_solution = false; ++ for (int itry = -3; itry <= 3; ++itry) { ++ quantizer.find_best_match(amax/(scale_0 + kStep*itry), xaux, weight, best_idx); ++ auto [d, score] = quantizer.find_best_scale(xaux, weight, best_idx); ++ if (score > best) { ++ best = score; ++ found_solution = true; ++ scales[ib] = d; ++ std::memcpy(best_idx+Q::kNg, best_idx, Q::kNg*sizeof(int)); ++ } ++ } ++ if (!found_solution) { ++ fprintf(stderr, "======================= %s: failed to find solution for a block\n", __func__); ++ fprintf(stderr, "Model weights and importances:\n"); ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ fprintf(stderr, "%2d %g %g\n", j, xaux[j], weight[j]); ++ } ++ GGML_ASSERT(false); ++ } ++ ++ auto xt = qtmp + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ auto q = quantizer.values() + Q::kGroupSize*best_idx[Q::kNg+ig]; ++ for (int j = 0; j < Q::kGroupSize; ++j) *xt++ = q[j]; ++ } ++ ++ float abs_scale = std::abs(scales[ib]); ++ if (abs_scale > amax_scale) { ++ amax_scale = abs_scale; ++ max_scale = scales[ib]; ++ } ++ } ++ ++ } ++ ++ GGML_ASSERT(max_scale >= 0); ++ float d = max_scale/15; ++ float best = 0; ++ for (int itry = -9; itry <= 9; ++itry) { ++ float id = (itry*0.2f + 15)/max_scale; ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ const float * xb = x + ibl*Q::kSuperBlockSize; ++ const float * qb = qtmp + ibl*Q::kSuperBlockSize; ++ const float * wb = all_weights + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ int ls = nearest_int(id*scales[ib]); ++ ls = std::max(0, std::min(15, ls)); ++ float dl = ls; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ float q = dl*qb[j]; ++ sumqx += wb[j]*std::abs(xb[j])*q; ++ sumq2 += wb[j]*q*q; ++ } ++ xb += Q::kBlockSize; ++ wb += Q::kBlockSize; ++ qb += Q::kBlockSize; ++ } ++ } ++ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { ++ d = sumqx/sumq2; best = d*sumqx; ++ } ++ } ++ ++ float id = d ? 1/d : 0.f; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ auto scales = all_scales + ibl*Q::kNblock; ++ for (int ib = 0; ib < Q::kNblock/2; ++ib) { ++ int ls1 = nearest_int(id*scales[ib]); ++ int ls2 = nearest_int(id*scales[ib + Q::kNblock/2]); ++ ls1 = std::max(0, std::min(15, ls1)); ++ ls2 = std::max(0, std::min(15, ls2)); ++ y[ibl].scales[ib] = ls1 | (ls2 << 4); ++ } ++ } ++ ++ *dptr = d; ++ ++ for (int iloop = 0; iloop < 1; ++iloop) { ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ uint16_t * ql = (uint16_t *)y[ibl].ql; ++ ++ std::memset(y[ibl].qh, 0, kNumGroups/2); ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * xb = xbl + Q::kBlockSize*ib; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ xaux[j] = std::abs(xb[j]); ++ if (xb[j] < 0) y[ibl].qh[j] |= (1 << ib); ++ } ++ int ls = (y[ibl].scales[ib%(Q::kNblock/2)] >> 4*(ib/(Q::kNblock/2))) & 0xf; ++ float dl = d*ls; ++ quantizer.find_best_match(dl, xaux, weight, best_idx); ++ ++ for (int j = 0; j < Q::kNg; ++j) { ++ ql[ib*Q::kNg+j] = best_idx[j]; ++ auto xl = xaux + Q::kGroupSize*j; ++ auto wl = weight + Q::kGroupSize*j; ++ auto ql = quantizer.values() + best_idx[j]*Q::kGroupSize; ++ for (int k = 0; k < Q::kGroupSize; ++k) { ++ float q = ql[k]*ls; ++ sumqx += wl[k]*xl[k]*q; ++ sumq2 += wl[k]*q*q; ++ } ++ } ++ } ++ } ++ if (sumq2 > 0) { ++ d = sumqx/sumq2; ++ *dptr = d; ++ if (!d) break; ++ } else { ++ break; ++ } ++ } ++} ++} ++ ++void quantize_row_iq3_kt_ref(const float * x, block_iq3_kt * y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq3_kt(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq3_kt(const float * x, void * vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq3_kt * y = (block_iq3_kt *)vy; ++ quantize_row_iq3_kt_ref(x, y, k); ++} ++ ++size_t quantize_iq3_kt(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ3_KT, n_per_row); ++ std::vector scales(n_per_row/QuantizerIQ3KT::kBlockSize); ++ std::vector weights(n_per_row), xtmp(n_per_row); ++ char * qrow = (char *)dst; ++ for (int64_t row = 0; row < nrows; ++row) { ++ quantize_row_iq3_kt_impl(src, (void *)qrow, n_per_row, imatrix, scales.data(), weights.data(), xtmp.data()); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq3_kt(const block_iq3_kt * x, float * y, int64_t k) { ++#ifdef __AVX2__ ++ //if (iqk_dequantize_ktquants(GGML_TYPE_IQ3_KT, k, x, 0, y, 0, 1)) return; ++#endif ++ using Q = QuantizerIQ3KT; ++ constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize; ++ assert(k % Q::kSuperBlockSize == 0); ++ const int nb = k / Q::kSuperBlockSize; ++ const float * dptr = (const float *)x; ++ const float d = *dptr * Q::kScale; ++ x = (const block_iq3_kt *)(dptr + 1); ++ auto& deq = iq3kt_quantizer(); ++ for (int ibl = 0; ibl < nb; ++ibl) { ++ auto yl = y + ibl*Q::kSuperBlockSize; ++ auto yh = yl + Q::kSuperBlockSize/2; ++ auto qll = (const uint16_t *)x[ibl].ql; ++ auto qlh = qll + kNumGroups/2; ++ int jj = 0; ++ for (int ib = 0; ib < Q::kNblock/2; ++ib) { ++ float sl = d * (x[ibl].scales[ib] & 0xf); ++ float sh = d * (x[ibl].scales[ib] >> 4); ++ uint8_t l_mask = 1 << ib; ++ uint8_t h_mask = l_mask << (Q::kNblock/2); ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ deq.set_values(qll[jj], yl, sl); ++ deq.set_values(qlh[jj], yh, sh); ++ for (int j = 0; j < Q::kGroupSize; ++j) { ++ if (x[ibl].qh[ig*Q::kGroupSize+j] & l_mask) yl[j] = -yl[j]; ++ if (x[ibl].qh[ig*Q::kGroupSize+j] & h_mask) yh[j] = -yh[j]; ++ } ++ yl += Q::kGroupSize; ++ yh += Q::kGroupSize; ++ ++jj; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq3_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ3_KT, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++} ++ ++// ======================================== iq4_kt ++ ++namespace{ ++ ++using QuantizerIQ4KT = QuantizerIQKT<32, 4, 15, false, true>; ++ ++const QuantizerIQ4KT& iq4kt_quantizer(bool with_offset = false) { ++ static std::mutex mutex; ++ std::lock_guard lock(mutex); ++ static std::unique_ptr quantizer1; ++ static std::unique_ptr quantizer2; ++ if (with_offset) { ++ if (!quantizer2) quantizer2 = std::make_unique(625, 6, 4096+32768); ++ return *quantizer2; ++ } ++ if (!quantizer1) quantizer1 = std::make_unique(625, 6, 4096); ++ return *quantizer1; ++} ++ ++const QuantizerIQ4KT& iq4kt_dequantizer() { ++ static std::mutex mutex; ++ std::lock_guard lock(mutex); ++ static std::unique_ptr dequantizer; ++ if (!dequantizer) dequantizer = std::make_unique(0, 0, 4096); ++ return *dequantizer; ++} ++ ++void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights) { ++ ++ constexpr float kSigmaScale = 2.0f; ++ constexpr int kNtry = 2; ++ using Q = QuantizerIQ4KT; ++ ++ static_assert(Q::kNumVal%8 == 0); ++ ++ float * dptr = (float *)vy; ++ ++ block_iq4_kt * y = (block_iq4_kt *)(dptr + 1); ++ ++ auto& quantizer1 = iq4kt_quantizer(); ++ auto& quantizer2 = iq4kt_quantizer(true); ++ ++ int nblock = n_per_row / Q::kSuperBlockSize; ++ ++ Q::set_weights(kSigmaScale, nblock, x, quant_weights, all_weights); ++ ++ float amax_row = 0; ++ for (int j = 0; j < n_per_row; ++j) { ++ amax_row = std::max(amax_row, std::abs(x[j])); ++ } ++ if (!amax_row) { ++ dptr[0] = 0.f; ++ std::memset(y, 0, nblock*sizeof(block_iq4_kt)); ++ return; ++ } ++ ++ int best_idx[2*Q::kNg]; ++ float xaux[Q::kBlockSize]; ++ ++ float amax_scale = 0, max_scale = 0; ++ ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ memset(&y[ibl], 0, sizeof(block_iq4_kt)); ++ ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ float amax = 0; ++ for (int j = 0; j < Q::kBlockSize; ++j) { ++ xaux[j] = xbl[ib*Q::kBlockSize+j]; ++ float ax = std::abs(xaux[j]); ++ amax = std::max(amax, ax); ++ } ++ if (amax < 1e-16f) { ++ scales[ib] = 0; ++ continue; ++ } ++ float best = 0; ++ float scale_0 = std::max(90.f, 124.f*amax/amax_row); ++ for (int itry = -kNtry; itry <= kNtry; ++itry) { ++ quantizer1.find_best_match( amax/(8.f*itry + scale_0), xaux, weight, best_idx); ++ auto [dp, score_p] = quantizer1.find_best_scale(xaux, weight, best_idx); ++ if (score_p > best) { ++ best = score_p; scales[ib] = dp; ++ } ++ quantizer1.find_best_match(-amax/(8.f*itry + scale_0), xaux, weight, best_idx); ++ auto [dm, score_m] = quantizer1.find_best_scale(xaux, weight, best_idx); ++ if (score_m > best) { ++ best = score_m; scales[ib] = dm; ++ } ++ } ++ ++ quantizer2.find_best_match(scales[ib], xaux, weight, best_idx); ++ auto [d, score] = quantizer2.find_best_scale(xaux, weight, best_idx); ++ if (score > best) { ++ scales[ib] = d; ++ y[ibl].qs[ib] = 1; ++ } ++ bool with_offset = false; ++ for (int itry = -kNtry; itry <= kNtry; ++itry) { ++ quantizer2.find_best_match( amax/(8.f*itry + scale_0), xaux, weight, best_idx); ++ auto [dp, score_p] = quantizer2.find_best_scale(xaux, weight, best_idx); ++ if (score_p > best) { ++ best = score_p; scales[ib] = dp; with_offset = true; ++ } ++ quantizer2.find_best_match(-amax/(8.f*itry + scale_0), xaux, weight, best_idx); ++ auto [dm, score_m] = quantizer2.find_best_scale(xaux, weight, best_idx); ++ if (score_m > best) { ++ best = score_m; scales[ib] = dm; with_offset = true; ++ } ++ } ++ if (with_offset) y[ibl].qs[ib] = 1; ++ ++ float abs_scale = std::abs(scales[ib]); ++ if (abs_scale > amax_scale) { ++ amax_scale = abs_scale; ++ max_scale = scales[ib]; ++ } ++ } ++ ++ } ++ ++ float d = -max_scale/64; ++ ++ dptr[0] = d; ++ if (!d) return; ++ ++ constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize; ++ ++ for (int iloop = 0; iloop < 1; ++iloop) { ++ ++ const float id = 1/d; ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int ibl = 0; ibl < nblock; ++ibl) { ++ ++ // high 3 bits + scales ++ // each block of 32 needs 8 x 3 (high bits) + 1 x 8 (scale) = 32 bits = 1 x uint32_t ++ // we have 8 blocks ++ auto shb = y[ibl].qs; // high 3 bits + scales ++ auto ql = (uint8_t *)(shb + Q::kNblock); ++ auto qh = ql + kNumGroups; ++ std::memset(qh, 0, kNumGroups/2); ++ const float * xbl = x + ibl*Q::kSuperBlockSize; ++ auto scales = all_scales + ibl*Q::kNblock; ++ ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ auto& quantizer = y[ibl].qs[ib] & 1 ? quantizer2 : quantizer1; ++ const float * weight = all_weights + ibl*Q::kSuperBlockSize + ib*Q::kBlockSize; ++ for (int j = 0; j < Q::kBlockSize; ++j) xaux[j] = xbl[ib*Q::kBlockSize+j]; ++ int ls = nearest_int(id*scales[ib]); ++ ls = std::min(ls, 63); ++ *(uint8_t *)(shb + ib) = ((ls + 64) << 1) | (shb[ib] & 1); ++ float dl = d*ls; ++ quantizer.find_best_match(dl, xaux, weight, best_idx); ++ ++ for (int j = 0; j < Q::kNg; ++j) { ++ shb[ib] |= ((best_idx[j] >> 12) << (8 + 3*j)); ++ ql[Q::kNg*ib + j] = best_idx[j] & 255; ++ qh[(Q::kNg*ib + j)%(kNumGroups/2)] |= ((best_idx[j] >> 8) & 0xf) << 4*((Q::kNg*ib + j)/(kNumGroups/2)); ++ auto xl = xaux + Q::kGroupSize*j; ++ auto wl = weight + Q::kGroupSize*j; ++ auto ql = quantizer.values() + Q::kGroupSize*best_idx[j]; ++ for (int k = 0; k < Q::kGroupSize; ++k) { ++ float q = ql[k]*ls; ++ sumqx += wl[k]*xl[k]*q; ++ sumq2 += wl[k]*q*q; ++ } ++ } ++ } ++ } ++ if (sumq2 > 0) { ++ d = sumqx/sumq2; ++ dptr[0] = d; ++ if (!d) break; ++ } else { ++ break; ++ } ++ } ++} ++} ++ ++void quantize_row_iq4_kt_ref(const float * GGML_RESTRICT x, block_iq4_kt * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_K == 0); ++ quantize_iq4_kt(x, (void *)y, 1, k, nullptr, nullptr); ++} ++ ++void quantize_row_iq4_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { ++ assert(k % QK_K == 0); ++ block_iq4_kt * y = (block_iq4_kt *)vy; ++ quantize_row_iq4_kt_ref(x, y, k); ++} ++ ++size_t quantize_iq4_kt(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row%QK_K == 0); ++ auto row_size = ggml_row_size(GGML_TYPE_IQ4_KT, n_per_row); ++ std::vector scales(n_per_row/QuantizerIQ4KT::kBlockSize); ++ std::vector weights(n_per_row); ++ char * qrow = (char *)dst; ++ for (int64_t row = 0; row < nrows; ++row) { ++ quantize_row_iq4_kt_impl(src, (void *)qrow, n_per_row, imatrix, scales.data(), weights.data()); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrows * row_size; ++} ++ ++void dequantize_row_iq4_kt(const block_iq4_kt * x, float * y, int64_t k) { ++#ifdef __AVX2__ ++ //if (iqk_dequantize_ktquants(GGML_TYPE_IQ4_KT, k, x, 0, y, 0, 1)) return; ++#endif ++ using Q = QuantizerIQ4KT; ++ assert(k % Q::kSuperBlockSize == 0); ++ constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize; ++ const int nb = k / Q::kSuperBlockSize; ++ const float * dptr = (const float *)x; ++ const float d = dptr[0] * Q::kScale; ++ x = (const block_iq4_kt *)(dptr + 1); ++ auto& deq = iq4kt_dequantizer(); ++ for (int ibl = 0; ibl < nb; ++ibl) { ++ auto shb = x[ibl].qs; ++ auto ql = (const uint8_t *)(shb + Q::kNblock); ++ auto qh = ql + kNumGroups; ++ for (int ib = 0; ib < Q::kNblock; ++ib) { ++ int offset = shb[ib] & 1 ? 32768 + 4096 : 4096; ++ int ls = int((shb[ib] & 0xff) >> 1) - 64; ++ float sl = d * ls; ++ for (int ig = 0; ig < Q::kNg; ++ig) { ++ int jj = ib*Q::kNg+ig; ++ uint16_t idx = ql[jj] | ((qh[jj%(kNumGroups/2)] << (8 - 4*(jj/(kNumGroups/2)))) & 0xf00) | (((shb[ib] >> (8 + 3*ig)) & 7) << 12); ++ deq.set_values(idx, y, sl, offset); ++ y += Q::kGroupSize; ++ } ++ } ++ } ++} ++ ++void vec_dot_iq4_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK_K == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ ++#if GGML_USE_IQK_MULMAT ++ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ4_KT, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) { ++ return; ++ } ++#endif ++ ++} ++ ++void quantize_row_q1_0_g128_ref(const float * x, block_q1_0_g128 * y, int64_t k) { ++ quantize_row_q1_0_g128(x, y, k); ++} ++ ++void quantize_row_q1_0_g128(const float * x, void * vy, int64_t k) { ++ assert(k % QK1_0_G128 == 0); ++ int nb = k / QK1_0_G128; ++ auto y = (block_q1_0_g128 *)vy; ++ for (int ib = 0; ib < nb; ++ib) { ++ float sum = 0; ++ for (int j = 0; j < QK1_0_G128; ++j) sum += std::abs(x[j]); ++ float d = sum / QK1_0_G128; ++ y[ib].d = GGML_FP32_TO_FP16(d); ++ std::memset(y[ib].qs, 0, QK1_0_G128/8); ++ for (int j = 0; j < QK1_0_G128; ++j) { ++ if (x[j] >= 0.0f) { ++ y[ib].qs[j / 8] |= (1 << (j % 8)); ++ } ++ } ++ x += QK1_0_G128; ++ } ++} ++ ++size_t quantize_q1_0_g128(const float * src, void * dst, int64_t nrows, int64_t n_per_row, [[maybe_unused]] const float * imatrix, ++ [[maybe_unused]] const quantize_user_data * user_data) { ++ GGML_ASSERT(n_per_row % QK1_0_G128 == 0); ++ int64_t ntot = nrows * n_per_row; ++ quantize_row_q1_0_g128(src, dst, ntot); ++ int64_t nblock = ntot / QK1_0_G128; ++ return nblock * sizeof(block_q1_0_g128); ++} ++ ++void dequantize_row_q1_0_g128(const block_q1_0_g128 * x, float * y, int64_t k) { ++ assert(k % QK1_0_G128 == 0); ++ constexpr uint8_t k_mask[8] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80}; ++ int nb = k / QK1_0_G128; ++ for (int ib = 0; ib < nb; ++ib) { ++ float d = GGML_FP16_TO_FP32(x[ib].d); ++ for (int i = 0; i < QK1_0_G128/8; ++i) { ++ for (int j = 0; j < 8; ++j) { ++ *y++ = x[ib].qs[i] & k_mask[j] ? d : -d; ++ } ++ } ++ } ++} ++ ++void vec_dot_q1_0_g128_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) { ++ assert(n % QK1_0_G128 == 0); ++ assert(nrc == 1); ++ GGML_UNUSED(nrc); ++ GGML_UNUSED(bx); ++ GGML_UNUSED(by); ++ GGML_UNUSED(bs); ++ int nb = n / QK1_0_G128; ++ ++ constexpr uint8_t k_mask[8] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80}; ++ ++ constexpr int n4 = QK1_0_G128 / QK8_0; ++ ++ auto x = (const block_q1_0_g128 *)vx; ++ auto y = (const block_q8_0_x4 *)vy; ++ int16_t sumi[QK1_0_G128/8]; ++ float sumf = 0; ++ for (int ib = 0; ib < nb; ++ib) { ++ auto dx = GGML_FP16_TO_FP32(x[ib].d); ++ auto qx = x[ib].qs; ++ auto qy = y[ib].qs; ++ for (int k = 0; k < QK1_0_G128/8; ++k) { ++ uint8_t bits = qx[k]; ++ int16_t s = 0; ++ for (int j = 0; j < 8; ++j) { ++ s += (bits & k_mask[j] ? qy[j] : -qy[j]); ++ } ++ qy += 8; ++ sumi[k] = s; ++ } ++ auto s = sumi; ++ for (int k = 0; k < n4; ++k) { ++ float dy = GGML_FP16_TO_FP32(y[ib].d[k]); ++ sumf += dx*dy*(s[0] + s[1] + s[2] + s[3]); ++ s += 4; ++ } ++ } ++ *s = sumf; ++} ++ ++namespace { ++template ++inline int check_row_for_blocks_256_fp16(int nblock, const Block * x) { ++ int nbad = 0; ++ for (int ib = 0; ib < nblock; ++ib) { ++ float d = GGML_FP16_TO_FP32(x[ib].d); ++ if (isnan(d)) ++nbad; ++ } ++ return nbad; ++} ++template ++bool check_tensor_for_blocks_256_fp16(const ggml_tensor * tensor) { ++ int nblock = tensor->ne[0]/QK_K; ++ int nbad = 0; ++ for (int row = 0; row < ggml_nrows(tensor); ++row) { ++ auto x = (const Block *)((const char *)tensor->data + tensor->nb[1]*row); ++ nbad += check_row_for_blocks_256_fp16(nblock, x); ++ } ++ if (nbad > 0) { ++ fprintf(stderr, "%s: found %d NaN block scales out of %g blocks in tensor %s\n", __func__, ++ nbad, 1.*ggml_nrows(tensor)*nblock, tensor->name); ++ if (tensor->ne[2] > 1) { ++ int nb = tensor->ne[0]/QK_K; ++ for (int64_t i02 = 0; i02 < tensor->ne[2]; ++i02) { ++ int nbad_expert = 0; ++ auto xex = (const char *)((const char *)tensor->data + i02*tensor->nb[2]); ++ for (int64_t i01 = 0; i01 < tensor->ne[1]; ++i01) { ++ auto xr = (const Block *)(xex + i01*tensor->nb[1]); ++ nbad_expert += check_row_for_blocks_256_fp16(nb, xr); ++ } ++ if (nbad_expert > 0) fprintf(stderr," there are %d NaN block scales for expert %g\n", nbad_expert, 1.*i02); ++ } ++ } ++ return false; ++ } ++ return true; ++} ++template ++inline int check_row_for_blocks_256_fp16(int nblock, const Block * x, int nr) { ++ int nbad = 0; ++ for (int ib = 0; ib < nblock; ++ib) { ++ for (int j = 0; j < nr; ++j) { ++ if (!isfinite(GGML_FP16_TO_FP32(x[ib].d[j]))) ++nbad; ++ } ++ } ++ return nbad; ++} ++template ++bool check_tensor_for_blocks_256_fp16_repacked(const ggml_tensor * tensor) { ++ int nblock = tensor->ne[0]/QK_K; ++ int nbad = 0; ++ for (int row = 0; row < ggml_nrows(tensor); row += nr) { ++ auto x = (const Block *)((const char *)tensor->data + tensor->nb[1]*row); ++ nbad += check_row_for_blocks_256_fp16(nblock, x, nr); ++ } ++ if (nbad > 0) { ++ fprintf(stderr, "%s: found %d NaN block scales out of %g blocks in tensor %s\n", __func__, ++ nbad, 1.*ggml_nrows(tensor)*nblock, tensor->name); ++ if (tensor->ne[2] > 1) { ++ int nb = tensor->ne[0]/QK_K; ++ for (int64_t i02 = 0; i02 < tensor->ne[2]; ++i02) { ++ int nbad_expert = 0; ++ auto xex = (const char *)((const char *)tensor->data + i02*tensor->nb[2]); ++ for (int64_t i01 = 0; i01 < tensor->ne[1]; i01 += nr) { ++ auto xr = (const Block *)(xex + i01*tensor->nb[1]); ++ nbad_expert += check_row_for_blocks_256_fp16(nb, xr, nr); ++ } ++ if (nbad_expert > 0) fprintf(stderr," there are %d NaN block scales for expert %g\n", nbad_expert, 1.*i02); ++ } ++ } ++ return false; ++ } ++ return true; ++} ++struct F32Scale { ++ static inline int check_row(const char * data) { ++ float d = *(const float *)data; ++ return isfinite(d) ? 0 : 1; ++ } ++}; ++struct F16Scale { ++ static inline int check_row(const char * data) { ++ float d = GGML_FP16_TO_FP32(*(const ggml_half *)data); ++ return isfinite(d) ? 0 : 1; ++ } ++}; ++template ++struct F32ScaleRX { ++ static inline int check_row(const char * data) { ++ auto d = (const float *)data; ++ int nbad = 0; ++ for (int i = 0; i < nr; ++i) { ++ if (!isfinite(d[i])) ++nbad; ++ } ++ return nbad; ++ } ++}; ++template ++struct F16ScaleRX { ++ static inline int check_row(const char * data) { ++ auto d = (const ggml_half *)data; ++ int nbad = 0; ++ for (int i = 0; i < nr; ++i) { ++ if (!isfinite(GGML_FP16_TO_FP32(d[i]))) ++nbad; ++ } ++ return nbad; ++ } ++}; ++template ++bool check_tensor_row_scales(const ggml_tensor * tensor) { ++ auto row_size = ggml_row_size(tensor->type, tensor->ne[0]); ++ int num_rows = ggml_nrows(tensor); ++ auto data = (const char *)tensor->data; ++ int nbad = 0; ++ for (int row = 0; row < num_rows; ++row) { ++ nbad += RS::check_row(data); ++ data += row_size; ++ } ++ if (nbad > 0) { ++ fprintf(stderr, "%s: found %d NaN row scales out of %d rows in tensor %s\n", __func__, ++ nbad, num_rows, tensor->name); ++ return false; ++ } ++ return true; ++} ++} ++ ++bool iqk_validate_tensor(const ggml_tensor * tensor) { ++ if (!tensor) return true; ++ if (!ggml_is_contiguous(tensor)) return true; ++ ++ switch (tensor->type) { ++ case GGML_TYPE_IQ2_K: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ3_K: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ4_K: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ5_K: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ6_K: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ2_XXS: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ2_XS: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ2_S: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ3_XXS: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ3_S: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ4_XS: return check_tensor_for_blocks_256_fp16(tensor); ++ case GGML_TYPE_IQ2_K_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ3_K_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ4_K_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ5_K_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ2_XXS_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ2_XS_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ2_S_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ3_XXS_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ3_S_R4: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ4_XS_R8: return check_tensor_for_blocks_256_fp16_repacked(tensor); ++ case GGML_TYPE_IQ2_BN: ++ case GGML_TYPE_IQ4_KSS: ++ case GGML_TYPE_IQ4_KS: ++ case GGML_TYPE_IQ5_KS: return check_tensor_row_scales(tensor); ++ case GGML_TYPE_IQ2_BN_R4: ++ case GGML_TYPE_IQ4_KS_R4: ++ case GGML_TYPE_IQ5_KS_R4: return check_tensor_row_scales>(tensor); ++ case GGML_TYPE_IQ1_BN: ++ case GGML_TYPE_IQ2_KS: ++ case GGML_TYPE_IQ2_KL: ++ case GGML_TYPE_IQ3_KS: return check_tensor_row_scales(tensor); ++ case GGML_TYPE_IQ1_S_R4: ++ case GGML_TYPE_IQ1_M_R4: return check_tensor_row_scales>(tensor); ++ ++ default: break; ++ } ++ return true; ++} +diff --git a/llama.cpp/ggml/src/iqk/iqk_quantize.h b/llama.cpp/ggml/src/iqk/iqk_quantize.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_quantize.h +@@ -0,0 +1,385 @@ ++// opencoti F5-opt W2 (#290): the mxfp4-family quant entry points (quantize_mxfp4 ++// et al.) were renamed iqk_*_mxfp4 — ik_llama gives them a 6-arg (quantize_user_data) ++// signature that collides with llamafile's own 5-arg quantize_mxfp4 in ggml-quants.{h,c}. ++// We never call ik_llama's mxfp4 path; the rename keeps both vintages linkable. Re-sync: reapply. ++// ++// Copyright (C) 2024-2025 Iwan Kawrakow ++// MIT license ++// SPDX-License-Identifier: MIT ++// ++ ++#pragma once ++ ++#include ++#include ++ ++#define GGML_COMMON_DECL_C ++#include "ggml-common.h" ++ ++#ifdef __cplusplus ++#define GGML_RESTRICT ++extern "C" { ++#else ++#define GGML_RESTRICT restrict ++#endif ++ ++// opencoti F5-opt W2 (#290): ik_llama defines this in its own ggml/include/ggml.h ++// (a superset of llamafile's ggml.h, which we must not edit). iqk_quantize.cpp ++// dereferences user_data->slow_iq2_ks, so the struct must be COMPLETE here, not ++// merely forward-declared. Body copied verbatim from ik_llama ggml.h. ++#ifdef __cplusplus ++struct quantize_user_data { ++ bool symmetric_q4_0; ++ bool slow_iq2_ks; ++}; ++#else ++struct quantize_user_data; ++#endif ++ ++void quantize_row_iq2_k_ref(const float * GGML_RESTRICT x, block_iq2_k * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_k(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_k(const block_iq2_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_k_ref(const float * GGML_RESTRICT x, block_iq3_k * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_k(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_k(const block_iq3_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_ks_ref(const float * GGML_RESTRICT x, block_iq3_ks * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_ks(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_ks(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_ks(const block_iq3_ks * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_ks_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_k_ref(const float * GGML_RESTRICT x, block_iq4_k * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_k(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_k(const block_iq4_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq5_k_ref(const float * GGML_RESTRICT x, block_iq5_k * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq5_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq5_k(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq5_k(const block_iq5_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq5_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq6_k_ref(const float * GGML_RESTRICT x, block_iq6_k * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq6_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq6_k(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq6_k(const block_iq6_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq6_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_ks_ref(const float * GGML_RESTRICT x, block_iq4_ks * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_ks(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_ks(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_ks(const block_iq4_ks * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_ks_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_kss_ref(const float * GGML_RESTRICT x, block_iq4_kss * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_kss(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_kss(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_kss(const block_iq4_kss * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_kss_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void iqk_quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); ++void iqk_quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t iqk_quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void iqk_dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_mxfp4_q8_0_x4(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_ks_ref(const float * GGML_RESTRICT x, block_iq2_ks * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_ks(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_ks(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_ks(const block_iq2_ks * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_ks_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_kl_ref(const float * GGML_RESTRICT x, block_iq2_kl * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_kl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_kl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_kl(const block_iq2_kl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_kl_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq1_kt_ref(const float * GGML_RESTRICT x, block_iq1_kt * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq1_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq1_kt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq1_kt(const block_iq1_kt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq1_kt_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_kt_ref(const float * GGML_RESTRICT x, block_iq2_kt * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_kt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_kt(const block_iq2_kt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_kt_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_kt_ref(const float * GGML_RESTRICT x, block_iq3_kt * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_kt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_kt(const block_iq3_kt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_kt_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_kt_ref(const float * GGML_RESTRICT x, block_iq4_kt * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_kt(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_kt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_kt(const block_iq4_kt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_kt_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq5_ks_ref(const float * GGML_RESTRICT x, block_iq5_ks * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq5_ks(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq5_ks(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq5_ks(const block_iq5_ks * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq5_ks_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_nl_r4_ref(const float * GGML_RESTRICT x, block_iq4_nl_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_nl_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_nl_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_nl_r4(const block_iq4_nl_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_nl_r4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q4_0_r8_ref(const float * GGML_RESTRICT x, block_iq4_nl_r8 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q4_0_r8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q4_0_r8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q4_0_r8(const block_iq4_nl_r8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q4_0_r8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q8_0_r8_ref(const float * GGML_RESTRICT x, block_q8_0_r8 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_0_r8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q8_0_r8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q8_0_r8(const block_q8_0_r8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q8_0_r8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q5_0_r4_ref(const float * GGML_RESTRICT x, block_q5_0_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q5_0_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q5_0_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q5_0_r4(const block_q5_0_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q5_0_r4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q6_0_r4_ref(const float * GGML_RESTRICT x, block_q6_0_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q6_0_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q6_0_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q6_0_r4(const block_q6_0_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q6_0_r4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_xs_r8_ref(const float * GGML_RESTRICT x, block_iq4_xs_r8 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_xs_r8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_xs_r8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_xs_r8(const block_iq4_xs_r8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_xs_r8_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_bn_ref (const float * GGML_RESTRICT x, block_iq2_bn * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_bn (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void dequantize_row_iq2_bn (const block_iq2_bn * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_bn (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void vec_dot_iq2_bn_q8_K64(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_bn_r4_ref (const float * GGML_RESTRICT x, block_iq2_bn * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_bn_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void dequantize_row_iq2_bn_r4(const block_iq2_bn * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_bn_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void vec_dot_iq2_bn_r4_q8_K64(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q3_k_r4_ref(const float * GGML_RESTRICT x, block_q3_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q3_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q3_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q3_k_r4(const block_q3_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q3_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q2_k_r4_ref(const float * GGML_RESTRICT x, block_q2_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q2_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q2_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q2_k_r4(const block_q2_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q2_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q4_k_r4_ref(const float * GGML_RESTRICT x, block_q4_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q4_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q4_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q4_k_r4(const block_q4_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q4_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q5_k_r4_ref(const float * GGML_RESTRICT x, block_q5_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q5_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q5_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q5_k_r4(const block_q5_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q5_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q6_k_r4_ref(const float * GGML_RESTRICT x, block_q6_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q6_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q6_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q6_k_r4(const block_q6_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q6_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq5_k_r4_ref(const float * GGML_RESTRICT x, block_iq5_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq5_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq5_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq5_k_r4(const block_iq5_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq5_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_k_r4_ref(const float * GGML_RESTRICT x, block_iq4_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_k_r4(const block_iq4_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_k_r4_ref(const float * GGML_RESTRICT x, block_iq3_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_k_r4(const block_iq3_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_k_r4_ref(const float * GGML_RESTRICT x, block_iq2_k_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_k_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_k_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_k_r4(const block_iq2_k_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_k_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq4_ks_r4_ref(const float * GGML_RESTRICT x, block_iq4_ks_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq4_ks_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq4_ks_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq4_ks_r4(const block_iq4_ks_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq4_ks_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq5_ks_r4_ref(const float * GGML_RESTRICT x, block_iq5_ks_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq5_ks_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq5_ks_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq5_ks_r4(const block_iq5_ks_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq5_ks_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_xxs_r4_ref(const float * GGML_RESTRICT x, block_iq2_xxs_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_xxs_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_xxs_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_xxs_r4(const block_iq2_xxs_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_xxs_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_xs_r4_ref(const float * GGML_RESTRICT x, block_iq2_xs_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_xs_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_xs_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_xs_r4(const block_iq2_xs_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_xs_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq2_s_r4_ref(const float * GGML_RESTRICT x, block_iq2_s_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq2_s_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq2_s_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq2_s_r4(const block_iq2_s_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq2_s_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_xxs_r4_ref(const float * GGML_RESTRICT x, block_iq3_xxs_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_xxs_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_xxs_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_xxs_r4(const block_iq3_xxs_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_xxs_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq3_s_r4_ref(const float * GGML_RESTRICT x, block_iq3_s_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq3_s_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq3_s_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq3_s_r4(const block_iq3_s_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq3_s_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq1_s_r4_ref(const float * GGML_RESTRICT x, block_iq1_s_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq1_s_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq1_s_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq1_s_r4(const block_iq1_s_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq1_s_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_iq1_m_r4_ref(const float * GGML_RESTRICT x, block_iq1_m_r4 * GGML_RESTRICT y, int64_t k); ++void quantize_row_iq1_m_r4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_iq1_m_r4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_iq1_m_r4(const block_iq1_m_r4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_iq1_m_r4_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q8_k_r8_ref(const float * GGML_RESTRICT x, block_q8_k_r8 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_k_r8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q8_k_r8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q8_k_r8(const block_q8_k_r8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q8_k_r8_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q8_k_r16_ref(const float * GGML_RESTRICT x, block_q8_k_r16 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_k_r16(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q8_k_r16(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q8_k_r16(const block_q8_k_r16 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q8_k_r16_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q8_KV_ref(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_KV(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q8_KV(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q8_KV(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q8_KV_q8_KV(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q8_KV_r8_ref(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_KV_r8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q8_KV_r8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q8_KV_r8(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q8_KV_r8_q8_KV(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void quantize_row_q1_0_g128_ref(const float * GGML_RESTRICT x, block_q1_0_g128 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q1_0_g128(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++size_t quantize_q1_0_g128(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * use_data); ++void dequantize_row_q1_0_g128(const block_q1_0_g128 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++void vec_dot_q1_0_g128_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); ++ ++void iqk_quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); ++void quantize_row_q8_K64_ref(const float * GGML_RESTRICT x, block_q8_K64 * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_K64(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_K128(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_K16(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_K32(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_KR8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_0_x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_1_x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void quantize_row_q8_2_x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void iqk_quantize_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++ ++void repack_f32_bf16_r16 (const void * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row); ++void repack_bf16_bf16_r16(const void * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row); ++ ++void iqk_repack_tensor(struct ggml_tensor * tensor); ++bool iqk_modify_tensor(struct ggml_tensor * tensor); ++ ++int iqk_repacked_type(const struct ggml_tensor * tensor); // int instead of ggml_type so we don't need to include ggml.h ++bool iqk_should_modify_tensor(const struct ggml_tensor * tensor); ++ ++// So we can re-pack Microsoft's BitNet I2_S quants ++void dequantize_row_ms_i2s(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++ ++typedef void (*to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++typedef void (*from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++void iqk_quantize_any(int from_type, int to_type, ++ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, ++ uint64_t nb0, uint64_t nb1, uint64_t nb2, uint64_t nb3, ++ const void * GGML_RESTRICT x, void * GGML_RESTRICT y, void * work_buffer, ++ to_float_t to_float, from_float_t from_float, int ith, int nth); ++ ++bool iqk_validate_tensor(const struct ggml_tensor * src); ++ ++// opencoti F5-opt W2 (#290): ik_llama's base quantizers take a trailing 6th arg ++// (const quantize_user_data *) that llamafile's ggml-quants.h does NOT have on the ++// same names — declaring the 6-arg form on the bare name would be a "conflicting ++// declaration" against the 5-arg upstream prototypes. So the iqk side renames these ++// calls to iqkbase_ (sed in iqk_quantize.cpp) and we declare the 6-arg ++// variants here. (iqk_ was avoided because iqk_quantize_q4_0 already names a ++// pre-existing 3-arg row helper.) quantize_q6_0 / iq1{s,m}_process_1block are IK-only ++// (absent upstream). Resolving these symbols is a later BUILD.mk/dispatch concern; ++// this only needs to compile. ++size_t iqkbase_quantize_q4_0 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q5_0 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q6_0 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q8_0 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q2_K (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q3_K (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q4_K (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q5_K (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_q6_K (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++size_t iqkbase_quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix, const struct quantize_user_data * user_data); ++void iqkbase_iq1s_process_1block(int block_size, const float * xb, const float * weight, int8_t * L, float * the_scale, uint16_t * the_index, int * the_shift, float * pairs, float * sumx, float * sumw); ++void iqkbase_iq1m_process_1block(const float * xb, const float * weight, int8_t * L, float * the_scale, uint16_t * the_index, int * the_shift, float * pairs); ++ ++#ifdef __cplusplus ++} ++#endif +diff --git a/llama.cpp/ggml/src/iqk/iqk_utils.h b/llama.cpp/ggml/src/iqk/iqk_utils.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/iqk/iqk_utils.h +@@ -0,0 +1,246 @@ ++#pragma once ++ ++#include "iqk_config.h" ++ ++#if defined IQK_IMPLEMENT ++ ++#include "ggml-impl.h" ++ ++#if defined(__ARM_NEON) && defined(__aarch64__) ++// copy-pasted from Justine Tunney's contribution to llama.cpp ++// adapted from arm limited optimized routine ++// the maximum error is 1.45358 plus 0.5 ulps ++// numbers above 88.38 will flush to infinity ++// numbers beneath -103.97 will flush to zero ++static inline float32x4_t v_expf(float32x4_t x) { ++ const float32x4_t r = vdupq_n_f32(0x1.8p23f); ++ const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f)); ++ const float32x4_t n = vsubq_f32(z, r); ++ const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n, ++ vdupq_n_f32(0x1.7f7d1cp-20f)); ++ const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23); ++ const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1)))); ++ const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126)); ++ const float32x4_t u = vmulq_f32(b, b); ++ const float32x4_t j = vfmaq_f32( ++ vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b), ++ vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b), ++ vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u); ++ if (!vpaddd_u64(vreinterpretq_u64_u32(c))) ++ return vfmaq_f32(k, j, k); ++ const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000)); ++ const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000))); ++ const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d)); ++ return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1), ++ vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j))); ++} ++static inline float16x8_t v_expf(float16x8_t x) { ++ auto val1 = v_expf(vcvt_f32_f16(vget_low_f16(x))); ++ auto val2 = v_expf(vcvt_f32_f16(vget_high_f16(x))); ++ return vcombine_f16(vcvt_f16_f32(val1), vcvt_f16_f32(val2)); ++} ++static inline float32x4_t v_tanh(float32x4_t x) { ++ const float32x4_t one = vdupq_n_f32(1.0f); ++ const float32x4_t two_x = vmulq_f32(x, vdupq_n_f32(2.f)); ++ const float32x4_t exp_two_x = v_expf(two_x); ++ const uint32x4_t mask = vcgtq_f32(x, vdupq_n_f32(10.f)); ++ const float32x4_t res = vdivq_f32(vsubq_f32(exp_two_x, one), vaddq_f32(exp_two_x, one)); ++ return vreinterpretq_f32_u32(vorrq_u32(vandq_u32(vreinterpretq_u32_f32(one), mask), vbicq_u32(vreinterpretq_u32_f32(res), mask))); ++ //return vdivq_f32(vsubq_f32(exp_two_x, one), vaddq_f32(exp_two_x, one)); ++} ++//inline float32x4_t v_tanh(float16x8_t x) { ++// auto val1 = v_tanh(vcvt_f32_f16(vget_low_f16(x))); ++// auto val2 = v_tanh(vcvt_f32_f16(vget_high_f16(x))); ++// return vcombine_f16(vcvt_f16_f32(val1), vcvt_f16_f32(val2)); ++//} ++static inline float32x4_t v_silu(float32x4_t x) { ++ const float32x4_t one = vdupq_n_f32(1.0f); ++ const float32x4_t zero = vdupq_n_f32(0.0f); ++ const float32x4_t neg_x = vsubq_f32(zero, x); ++ const float32x4_t exp_neg_x = v_expf(neg_x); ++ const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x); ++ return vdivq_f32(x, one_plus_exp_neg_x); ++} ++static inline float32x4_t v_silu_oai(float32x4_t x, float32x4_t alpha) { ++ const float32x4_t one = vdupq_n_f32(1.0f); ++ const float32x4_t neg_x = vmulq_f32(alpha, x); ++ const float32x4_t exp_neg_x = v_expf(neg_x); ++ const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x); ++ return vdivq_f32(x, one_plus_exp_neg_x); ++} ++static inline float32x4_t v_gelu(float32x4_t x, float32x4_t c1, float32x4_t c2) { ++ const float32x4_t one = vdupq_n_f32(1.0f); ++ float32x4_t arg = vfmaq_f32(one, c1, vmulq_f32(x, x)); ++ arg = vmulq_f32(arg, vmulq_f32(x, c2)); ++ float32x4_t exp_arg = v_expf(arg); ++ float32x4_t gelu = vmulq_f32(x, vdivq_f32(exp_arg, vaddq_f32(exp_arg, one))); ++ uint32x4_t mask = vcgtq_f32(x, vdupq_n_f32(10.f)); ++ return vbslq_f32(mask, x, gelu); ++} ++ ++#endif // __ARN_NEON ++ ++#if defined(__AVX512F__) && defined(_MSC_VER) ++#include ++ ++#ifndef __clang__ ++static inline __m512i operator|(__m512i a, __m512i b) { return _mm512_or_si512(a, b); } ++static inline __m512i operator&(__m512i a, __m512i b) { return _mm512_and_si512(a, b); } ++static inline __m512i operator^(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } ++#endif ++#endif ++ ++#if defined(__AVX512F__) && defined(__AVX512DQ__) ++ ++// copy-pasted from Justine Tunney's contribution to llama.cpp ++// adapted from arm limited optimized routine ++// the maximum error is 1.45358 plus 0.5 ulps ++// numbers above 88.38 will flush to infinity ++// numbers beneath -103.97 will flush to zero ++static inline __m512 v_expf(__m512 x) { ++ const __m512 r = _mm512_set1_ps(0x1.8p23f); ++ const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r); ++ const __m512 n = _mm512_sub_ps(z, r); ++ const __m512 b = ++ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f), ++ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x)); ++ const __mmask16 d = ++ _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ); ++ const __m512 u = _mm512_mul_ps(b, b); ++ const __m512 j = _mm512_fmadd_ps( ++ _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b, ++ _mm512_set1_ps(0x1.573e2ep-5f)), ++ u, ++ _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b, ++ _mm512_set1_ps(0x1.fffdb6p-2f))), ++ u, ++ _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F))); ++ const __m512 res = _mm512_scalef_ps(j, n); ++ if (_mm512_kortestz(d, d)) ++ return res; ++ const __m512 zero = _mm512_setzero_ps(); ++ const __m512 alt = _mm512_mask_blend_ps( ++ _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero); ++ return _mm512_mask_blend_ps(d, res, alt); ++} ++static inline __m512 v_tanh(__m512 x) { ++ const __m512 one = _mm512_set1_ps(1.0f); ++ const __m512 exp_two_x = v_expf(_mm512_mul_ps(x, _mm512_set1_ps(2.f))); ++ const __mmask16 mask = _mm512_cmp_ps_mask(x, _mm512_set1_ps(10.f), _CMP_GT_OQ); ++ const __m512 res = _mm512_div_ps(_mm512_sub_ps(exp_two_x, one), _mm512_add_ps(exp_two_x, one)); ++ return _mm512_mask_blend_ps(mask, res, one); ++} ++static inline __m512 v_gelu(__m512 x, __m512 c1, __m512 c2) { ++ const __m512 one = _mm512_set1_ps(1.0f); ++ __m512 arg = _mm512_fmadd_ps(x, _mm512_mul_ps(c1, x), one); ++ //__m512 arg = _mm512_add_ps(one, _mm512_mul_ps(_mm512_mul_ps(x, x), c1)); ++ arg = _mm512_mul_ps(arg, _mm512_mul_ps(c2, x)); ++ const __mmask16 mask = _mm512_cmp_ps_mask(arg, _mm512_set1_ps(30.f), _CMP_GT_OQ); ++ const __m512 exp_arg = v_expf(arg); ++ const __m512 ratio = _mm512_div_ps(exp_arg, _mm512_add_ps(exp_arg, one)); ++ return _mm512_mul_ps(x, _mm512_mask_blend_ps(mask, ratio, one)); ++} ++static inline __m512 v_silu(__m512 x) { ++ const __m512 one = _mm512_set1_ps(1); ++ const __m512 zero = _mm512_setzero_ps(); ++ const __m512 neg_x = _mm512_sub_ps(zero, x); ++ const __m512 exp_neg_x = v_expf(neg_x); ++ const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x); ++ return _mm512_div_ps(x, one_plus_exp_neg_x); ++} ++static inline __m512 v_silu_oai(__m512 x, __m512 alpha) { ++ const __m512 one = _mm512_set1_ps(1); ++ const __m512 neg_x = _mm512_mul_ps(alpha, x); ++ const __m512 exp_neg_x = v_expf(neg_x); ++ const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x); ++ return _mm512_div_ps(x, one_plus_exp_neg_x); ++} ++static inline __m512 v_clamp_max(__m512 x, __m512 max) { ++ auto mask = _mm512_cmp_ps_mask(x, max, _CMP_GT_OQ); ++ return _mm512_mask_blend_ps(mask, x, max); ++} ++#endif // __AVX512__ ++ ++#if defined(__AVX2__) && defined(__FMA__) ++ ++// adapted from arm limited optimized routine ++// the maximum error is 1.45358 plus 0.5 ulps ++// numbers above 88.38 will flush to infinity ++// numbers beneath -103.97 will flush to zero ++static inline __m256 v_expf(__m256 x) { ++ const __m256 r = _mm256_set1_ps(0x1.8p23f); ++ const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r); ++ const __m256 n = _mm256_sub_ps(z, r); ++ const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f), ++ _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x)); ++ const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23); ++ const __m256 k = _mm256_castsi256_ps( ++ _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1)))); ++ const __m256i c = _mm256_castps_si256( ++ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n), ++ _mm256_set1_ps(126), _CMP_GT_OQ)); ++ const __m256 u = _mm256_mul_ps(b, b); ++ const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b, ++ _mm256_set1_ps(0x1.573e2ep-5f)), u, ++ _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b, ++ _mm256_set1_ps(0x1.fffdb6p-2f))), ++ u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b)); ++ if (!_mm256_movemask_ps(_mm256_castsi256_ps(c))) ++ return _mm256_fmadd_ps(j, k, k); ++ const __m256i g = _mm256_and_si256( ++ _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)), ++ _mm256_set1_epi32(0x82000000u)); ++ const __m256 s1 = ++ _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u))); ++ const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g)); ++ const __m256i d = _mm256_castps_si256( ++ _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n), ++ _mm256_set1_ps(192), _CMP_GT_OQ)); ++ return _mm256_or_ps( ++ _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)), ++ _mm256_andnot_ps( ++ _mm256_castsi256_ps(d), ++ _mm256_or_ps( ++ _mm256_and_ps(_mm256_castsi256_ps(c), ++ _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)), ++ _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k))))); ++} ++static inline __m256 v_tanh(__m256 x) { ++ const __m256 one = _mm256_set1_ps(1.0f); ++ const __m256 exp_two_x = v_expf(_mm256_mul_ps(x, _mm256_set1_ps(2.f))); ++ const __m256 res = _mm256_div_ps(_mm256_sub_ps(exp_two_x, one), _mm256_add_ps(exp_two_x, one)); ++ const __m256 mask = _mm256_cmp_ps(x, _mm256_set1_ps(10.f), _CMP_GT_OQ); ++ return _mm256_or_ps(_mm256_and_ps(mask, one), _mm256_andnot_ps(mask, res)); ++} ++static inline __m256 v_gelu(__m256 x, __m256 c1, __m256 c2) { ++ const __m256 one = _mm256_set1_ps(1.0f); ++ const __m256 mask = _mm256_cmp_ps(x, _mm256_set1_ps(10.f), _CMP_GT_OQ); ++ __m256 arg = _mm256_add_ps(one, _mm256_mul_ps(_mm256_mul_ps(x, x), c1)); ++ arg = _mm256_mul_ps(arg, _mm256_mul_ps(x, c2)); ++ __m256 exp_arg = v_expf(arg); ++ __m256 gelu = _mm256_mul_ps(x, _mm256_div_ps(exp_arg, _mm256_add_ps(exp_arg, one))); ++ return _mm256_or_ps(_mm256_and_ps(mask, x), _mm256_andnot_ps(mask, gelu)); ++} ++static inline __m256 v_silu(__m256 x) { ++ const __m256 one = _mm256_set1_ps(1); ++ const __m256 zero = _mm256_setzero_ps(); ++ const __m256 neg_x = _mm256_sub_ps(zero, x); ++ const __m256 exp_neg_x = v_expf(neg_x); ++ const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x); ++ return _mm256_div_ps(x, one_plus_exp_neg_x); ++} ++static inline __m256 v_silu_oai(__m256 x, __m256 alpha) { ++ const __m256 one = _mm256_set1_ps(1); ++ const __m256 neg_x = _mm256_mul_ps(alpha, x); ++ const __m256 exp_neg_x = v_expf(neg_x); ++ const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x); ++ return _mm256_div_ps(x, one_plus_exp_neg_x); ++} ++static inline __m256 v_clamp_max(__m256 x, __m256 max) { ++ auto mask = _mm256_cmp_ps(x, max, _CMP_GT_OQ); ++ return _mm256_or_ps(_mm256_and_ps(mask, max), _mm256_andnot_ps(mask, x)); ++} ++ ++#endif // __AVX2__ ++ ++#endif // IQK_IMPLEMENT diff --git a/patches/0042-fused-moe.patch b/patches/0042-fused-moe.patch new file mode 100644 index 0000000000000000000000000000000000000000..20002e729c1d8db5726bea92d60e98865de0274d --- /dev/null +++ b/patches/0042-fused-moe.patch @@ -0,0 +1,527 @@ +opencoti F5-opt W3 (#292) — op-level fused MoE up+gate+GLU (GGML_OP_MOE_FUSED_UP_GATE) + +Adds a new ggml op, GGML_OP_MOE_FUSED_UP_GATE, that collapses a MoE FFN's two +separate per-expert projections (up = up_exps @ cur, gate = gate_exps @ cur) +plus the GLU activation into ONE op at decode (n_tokens == 1). The CUDA backend +dispatches it straight to the existing fused mmvq kernel (ggml_cuda_mul_mat_vec_q +with a fusion arg carrying the gate weight + glu_op), eliminating the gate_up +intermediate's HBM round-trip and the standalone GLU kernel launch per MoE layer +per decode token. + +Off by default. Engaged with --fused-moe-up-gate on (server) / cparams +.fused_moe_up_gate / adapter fusedMoeUpGate. The build_moe_ffn graph hook only +emits the op when the model presents SEPARATE up/gate expert weights +(gate_exps && !gate_up_exps), same type, SILU|GELU activation, at decode — i.e. +Qwen2-MoE / Qwen3-MoE / OLMoE / Mixtral-style layouts. Models that fuse gate+up +at the weight level (a single ffn_gate_up_exps — Gemma-4 A4B, Qwen3.5-MoE) +never match the hook and run unchanged. Flag-OFF is byte-identical to the pre-W3 +binary (op never emitted); flag-ON on a matching model is byte-identical to the +unfused two-matmul + GLU path. Validated on OLMoE-1B-7B-0924-Instruct-Q4_K_M: +graph-dump engagement confirmed (34 MOE_FUSED_UP_GATE nodes on CUDA0), greedy +decode byte-identical OFF vs ON, +2.4% decode tok/s (265.9 -> 272.2). + + opencoti-hook: f5-opt-fmoe — see docs/features/advanced_kv.md + +UPSTREAM PROVENANCE (for future re-sync) + Idea source : ik_llama.cpp op-level MoE fusion + https://github.com/ikawrakow/ik_llama.cpp + PR #229 (fused MoE mul_mat_id path), PR #520 (OOAE expert eval). + Commit context HEAD 8960c5ba5ee9db30ba838304373aa4dbec9f7cbd. + License MIT, Author Iwan Kawrakow. + Code vendored: NONE. This op is original opencoti code. ik_llama's + ggml_cuda_moe_up_gate_unary dispatches a fused mmvq-id kernel; + our tree already carries the equivalent primitive + (ggml_cuda_mul_mat_vec_q with ggml_cuda_mm_fusion_args.gate + + glu_op — ggml-cuda/mmvq.cu:555-582). W3 takes only the + op-level-fusion IDEA and routes our existing kernel through a + new ggml op + a build_moe_ffn graph hook. No ik_llama source is + copied into this patch. + ABI note : GGML_OP_COUNT 96 -> 97. The cosmocc binary AND the ggml-cuda.so + DSO must be rebuilt together and paired (asymmetric DSO cache + hygiene). The CPU backend deliberately aborts the op (CUDA-only; + CPU MoE fusion is out of scope — only the CPU embedder matters + for CPU speed, and that is the separate W2 iqk FA engine). + Op convention (locked from ggml-cuda/mmvq.cu): + out = up_proj(src0=up_exps @ src1=cur) (.) act(gate_proj=fusion.gate @ cur) + == ggml_geglu_split(gate, up); the kernel reads the GLU variant from + fusion.glu_op (NOT dst op_params). New op srcs: src0=up_exps, src1=gate_exps, + src2=cur, src3=ids; op_params[0]=glu_op. CUDA dispatch passes src0=up_exps, + src1=cur(=src2), ids=src3, fusion.gate=gate_exps(=src1), glu_op=op_params[0]. + +Files touched (all surgical; no new vendored files): + ggml/include/ggml.h enum GGML_OP_MOE_FUSED_UP_GATE + ggml_moe_up_gate decl + ggml/src/ggml.c NAME/SYMBOL entries, COUNT 96->97 asserts, creation fn + ggml/src/ggml-cpu/ggml-cpu.c forward-dispatch + n_tasks: CUDA-only ABORT stub + ggml/src/ggml-cuda/ggml-cuda.cu eval dispatch (reuse fused mmvq) + supports_op gate + src/llama-cparams.h bool fused_moe_up_gate + include/llama.h bool fused_moe_up_gate (llama_context_params) + src/llama-context.cpp param copy + default-params initializer + common/common.h bool fused_moe_up_gate = false + common/common.cpp cparams copy + common/arg.cpp --fused-moe-up-gate on|off (LLAMA_ARG_FUSED_MOE_UP_GATE) + src/llama-graph.cpp build_moe_ffn hook: emit op for separate-up/gate decode + +--- +opencoti note (2026-05-31, bug-250): hunks regenerated via snapshot-diff against the post-0040 chain so the patch strict git-applies (3 common/* hunks had whitespace context drift). Content unchanged. +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1375,6 +1375,91 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.slot_initial_ctx = value; + } + ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL})); ++ // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--rest-kv-eviction"}, ++ {"--no-rest-kv-eviction"}, ++ string_format("when context shift fires, evict the lowest-retention (KeyDiff key-similarity) contiguous window instead of the positional middle (default: %s)", params.rest_kv_eviction ? "enabled" : "disabled"), ++ [](common_params & params, bool value) { ++ params.rest_kv_eviction = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_EVICTION")); ++ add_opt(common_arg( ++ {"--rest-kv-recent"}, "N", ++ string_format("rest-kv: protect the most-recent N tokens from eviction (default: %d)", params.rest_kv_recent), ++ [](common_params & params, int value) { ++ params.rest_kv_recent = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_RECENT")); ++ add_opt(common_arg( ++ {"--rest-kv-layer"}, "N", ++ string_format("rest-kv: model layer whose keys score retention, -1 = auto/middle (default: %d)", params.rest_kv_layer), ++ [](common_params & params, int value) { ++ params.rest_kv_layer = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER")); ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--headinfer-gpu-heads-frac"}, "F", ++ string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac), ++ [](common_params & params, const std::string & value) { ++ params.headinfer_gpu_heads_frac = std::stof(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC")); ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--neo-pipeline"}, "MODE", ++ "neo (asymmetric GPU/CPU attention pipelining): off (default), on (engage when headinfer split is active for the layer), auto (reserved, same as on). Requires --headinfer-gpu-heads-frac < 1.0 to do anything.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off") params.neo_pipeline_mode = 0; ++ else if (value == "on") params.neo_pipeline_mode = 1; ++ else if (value == "auto") params.neo_pipeline_mode = 2; ++ else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); ++ // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--iqk-flash-attn"}, "MODE", ++ "iqk CPU flash-attention (vendored ik_llama.cpp engine): off (default), on. " ++ "Speeds up the M2 CPU-half attention on the token-generation path; falls back " ++ "to the mainline kernel for unsupported (type, shape) cases. x86_64-only.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.iqk_flash_attn = false; ++ else if (value == "on" || value == "true" || value == "1") params.iqk_flash_attn = true; ++ else throw std::invalid_argument("--iqk-flash-attn must be on|off"); ++ ggml_iqk_flash_attn_set_enabled(params.iqk_flash_attn); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_IQK_FLASH_ATTN")); ++ // opencoti F5-opt W3 (#292) fused-MoE — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--fused-moe-up-gate"}, "MODE", ++ "fused MoE up+gate+GLU (CUDA decode): off (default), on. Collapses the " ++ "separate-up/gate expert matmuls + GLU into one op at decode so the CUDA " ++ "backend runs the fused mmvq kernel without graph-pattern fusion. Only " ++ "engages for separate-up/gate MoE models (e.g. Gemma-4 A4B) at n_tokens==1.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.fused_moe_up_gate = false; ++ else if (value == "on" || value == "true" || value == "1") params.fused_moe_up_gate = true; ++ else throw std::invalid_argument("--fused-moe-up-gate must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_FUSED_MOE_UP_GATE")); ++ // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md ++ add_opt(common_arg( ++ {"--pcie-autodetect"}, "MODE", ++ "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.", ++ [](common_params & params, const std::string & value) { ++ if (value == "on" || value == "true" || value == "1") params.pcie_autodetect = true; ++ else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false; ++ else throw std::invalid_argument("--pcie-autodetect must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT")); ++ add_opt(common_arg( ++ {"--pcie-bw-gbps"}, "F", ++ string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps), ++ [](common_params & params, const std::string & value) { ++ params.pcie_bw_gbps = std::stof(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS")); + add_opt(common_arg( + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + {"--slot-shrink-idle-ms"}, "N", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1636,6 +1636,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + cparams.neo_pipeline_mode = params.neo_pipeline_mode; ++ // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md ++ cparams.fused_moe_up_gate = params.fused_moe_up_gate; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -445,6 +445,41 @@ struct common_params { + // active for the layer), 2 = auto (reserved for future load-based + // engagement; same as `on` today). + uint32_t neo_pipeline_mode = 0; ++ // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md ++ // false = off (default; unfused 3-node path). true = fuse decode MoE ++ // up+gate+GLU into one GGML_OP_MOE_FUSED_UP_GATE op for the CUDA backend. ++ bool fused_moe_up_gate = false; ++ // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md ++ // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 ++ // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / ++ // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default). ++ bool pcie_autodetect = true; ++ // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md ++ // Engage the vendored ik_llama.cpp CPU Flash-Attention engine on the M2 ++ // CPU-half (token-generation path). Off by default; flag-off is byte-identical. ++ // x86_64-only at runtime (the engine compiles only on the AVX2 x86_64 slice). ++ bool iqk_flash_attn = false; ++ float pcie_bw_gbps = 0.0f; ++ // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md ++ // Initial KV-cell allocation per sequence. 0 = full n_ctx_seq allocation ++ // (Phase 1 behavior — only zero-fill is deferred). Positive values cap ++ // the initial allocation and enable grow-on-demand up to n_ctx_seq. ++ int32_t slot_initial_ctx = 0; ++ // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md ++ // Idle threshold (ms) after which a grown soft cap is shrunk back to ++ // slot_initial_ctx and the over-cap pages are decommitted via ++ // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior). ++ int32_t slot_shrink_idle_ms = 0; ++ // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // Fraction of KV heads kept GPU-resident; the rest are offloaded to host ++ // memory and streamed back per step. 1.0 = off (no split). Only active for ++ // GPU-offloaded layers with flash-attention (non-transposed V cache). ++ float headinfer_gpu_heads_frac = 1.0f; ++ // opencoti F5 M3 neo — see docs/features/advanced_kv.md ++ // 0 = off (default; M2 concat path runs), 1 = on (engage when split ++ // active for the layer), 2 = auto (reserved for future load-based ++ // engagement; same as `on` today). ++ uint32_t neo_pipeline_mode = 0; + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -583,6 +583,8 @@ extern "C" { + + GGML_OP_GLU, + ++ GGML_OP_MOE_FUSED_UP_GATE, // opencoti-hook: f5-opt-fmoe — W3 #292 ++ + GGML_OP_COUNT, + }; + +@@ -1437,6 +1439,22 @@ extern "C" { + struct ggml_tensor * b, + struct ggml_tensor * ids); + ++ // opencoti-hook: f5-opt-fmoe — W3 #292: op-level fused MoE up+gate+GLU. ++ // build_moe_ffn otherwise emits {mul_mat_id(up), mul_mat_id(gate), glu} ++ // as 3 nodes that the CUDA backend can only fuse via the fragile ++ // ggml_can_fuse_subgraph graph-pattern path (which does not engage at ++ // decode for Gemma-4). This single op lets the backend run the already ++ // existing fused mmvq kernel directly. Output is GLU(gate@cur) (.) (up@cur), ++ // matching ggml_geglu_split(gate, up). up_exps/gate_exps: expert weight ++ // stacks (same type); cur: input activations; ids: expert selection. ++ GGML_API struct ggml_tensor * ggml_moe_up_gate( ++ struct ggml_context * ctx, ++ struct ggml_tensor * up_exps, ++ struct ggml_tensor * gate_exps, ++ struct ggml_tensor * cur, ++ struct ggml_tensor * ids, ++ enum ggml_glu_op glu_op); ++ + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -2066,6 +2066,13 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + { + ggml_compute_forward_glu(params, tensor); + } break; ++ case GGML_OP_MOE_FUSED_UP_GATE: ++ { ++ // opencoti-hook: f5-opt-fmoe — W3 #292. CUDA-only op. The ++ // build_moe_ffn graph hook emits it only when the CUDA backend ++ // owns the FFN nodes, so the CPU scheduler never reaches here. ++ GGML_ABORT("MOE_FUSED_UP_GATE has no CPU implementation (CUDA-only op)"); ++ } break; + case GGML_OP_GET_REL_POS: + { + ggml_compute_forward_get_rel_pos(params, tensor); +@@ -2333,6 +2340,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + GGML_ABORT("fatal error"); + } + break; ++ case GGML_OP_MOE_FUSED_UP_GATE: ++ { ++ // opencoti-hook: f5-opt-fmoe — W3 #292. CUDA-only; never ++ // scheduled on CPU. Single task keeps the planner well-formed. ++ n_tasks = 1; ++ } break; + case GGML_OP_SILU_BACK: + case GGML_OP_MUL: + case GGML_OP_DIV: +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -3005,6 +3005,25 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg + case GGML_OP_MUL_MAT_ID: + ggml_cuda_mul_mat_id(ctx, dst); + break; ++ case GGML_OP_MOE_FUSED_UP_GATE: ++ { ++ // opencoti-hook: f5-opt-fmoe — W3 #292. Drive the existing fused ++ // mmvq-id kernel directly: up = src0 @ src1, gate = fusion.gate @ ++ // src1, out = act(gate) (.) up. Byte-identical to the graph-pattern ++ // fusion path (see the { MUL_MAT_ID, MUL_MAT_ID, GLU } branch in ++ // the eval fusion loop). Decode-only (ne[2]==1): the build_moe_ffn ++ // hook only emits this at n_tokens==1 and supports_op enforces it. ++ const ggml_tensor * up_exps = dst->src[0]; ++ const ggml_tensor * gate_exps = dst->src[1]; ++ const ggml_tensor * cur = dst->src[2]; ++ const ggml_tensor * ids = dst->src[3]; ++ GGML_ASSERT(dst->ne[2] == 1 && "MOE_FUSED_UP_GATE is decode-only (n_tokens==1)"); ++ ggml_cuda_mm_fusion_args_host fusion_data{}; ++ fusion_data.gate = gate_exps; ++ fusion_data.glu_op = (ggml_glu_op) dst->op_params[0]; ++ ggml_cuda_mul_mat_vec_q(ctx, up_exps, cur, ids, dst, &fusion_data); ++ } ++ break; + case GGML_OP_OUT_PROD: + ggml_cuda_out_prod(ctx, dst); + break; +@@ -5210,6 +5229,44 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + return false; + } + break; ++ case GGML_OP_MOE_FUSED_UP_GATE: ++ { ++ // opencoti-hook: f5-opt-fmoe — W3 #292. Supported only in the ++ // exact shape the fused mmvq-id decode kernel handles: quantized ++ // up/gate expert stacks of identical type, F32 activations, I32 ++ // ids, single-token (decode) output, non-split weights, and a GLU ++ // op the kernel implements explicitly (GEGLU/SWIGLU/SWIGLU_OAI — ++ // others silently degrade to a plain product). build_moe_ffn only ++ // emits the op when all of these hold, so supports_op and the hook ++ // stay consistent (the scheduler never strands it on the CPU). ++ const ggml_tensor * up_exps = op->src[0]; ++ const ggml_tensor * gate_exps = op->src[1]; ++ const ggml_tensor * cur = op->src[2]; ++ const ggml_tensor * ids = op->src[3]; ++ if (!up_exps || !gate_exps || !cur || !ids) { ++ return false; ++ } ++ if (up_exps->buffer && ggml_backend_buft_is_cuda_split(up_exps->buffer->buft)) { ++ return false; ++ } ++ if (!ggml_is_quantized(up_exps->type) || up_exps->type != gate_exps->type) { ++ return false; ++ } ++ if (cur->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) { ++ return false; ++ } ++ if (op->ne[2] != 1) { ++ return false; ++ } ++ switch ((ggml_glu_op) op->op_params[0]) { ++ case GGML_GLU_OP_GEGLU: ++ case GGML_GLU_OP_SWIGLU: ++ case GGML_GLU_OP_SWIGLU_OAI: ++ return true; ++ default: ++ return false; ++ } ++ } + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + { +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -1078,9 +1078,10 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { + "OPT_STEP_SGD", + + "GLU", ++ "MOE_FUSED_UP_GATE", + }; + +-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96"); ++static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97"); + + static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "none", +@@ -1188,9 +1189,10 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "sgd(x)", + + "glu(x)", ++ "moe_up_gate(up,gate,x)", + }; + +-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96"); ++static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97"); + + static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); + +@@ -3314,6 +3316,53 @@ struct ggml_tensor * ggml_mul_mat_id( + return result; + } + ++// ggml_moe_up_gate ++// opencoti-hook: f5-opt-fmoe — W3 #292. Op-level fusion of the two parallel ++// expert matmuls (up = up_exps @ cur, gate = gate_exps @ cur) plus the GLU ++// activation that build_moe_ffn otherwise emits as 3 separate graph nodes. ++// Output == ggml_geglu_split(gate, up) == act(gate) (.) up, where the CUDA ++// backend runs the existing fused mmvq kernel (act on the gate projection, ++// multiply by the up projection — see ggml-cuda/mmvq.cu). Mirrors ++// ggml_mul_mat_id's shape contract: up_exps is the primary "as" weight, cur ++// the "b" activations shared by both projections, ids the expert selection. ++ ++struct ggml_tensor * ggml_moe_up_gate( ++ struct ggml_context * ctx, ++ struct ggml_tensor * up_exps, ++ struct ggml_tensor * gate_exps, ++ struct ggml_tensor * cur, ++ struct ggml_tensor * ids, ++ enum ggml_glu_op glu_op) { ++ GGML_ASSERT(!ggml_is_transposed(up_exps)); ++ GGML_ASSERT(!ggml_is_transposed(gate_exps)); ++ GGML_ASSERT(ids->type == GGML_TYPE_I32); ++ ++ // up and gate are parallel expert stacks: identical shape + type so one ++ // shared-activation fused mmvq can drive both weight sets. ++ GGML_ASSERT(ggml_are_same_shape(up_exps, gate_exps)); ++ GGML_ASSERT(up_exps->type == gate_exps->type); ++ ++ GGML_ASSERT(up_exps->ne[3] == 1); // 3d (one matrix per expert) ++ GGML_ASSERT(cur->ne[3] == 1); // cur is 3d ++ GGML_ASSERT(ids->ne[2] == 1 && ids->ne[3] == 1); // ids is 2d ++ GGML_ASSERT(ids->ne[1] == cur->ne[2]); // one expert list per cur row ++ GGML_ASSERT(up_exps->ne[0] == cur->ne[0]); // can_mul_mat ++ GGML_ASSERT(ids->ne[0] % cur->ne[1] == 0); // can broadcast ++ ++ const int64_t ne[4] = { up_exps->ne[1], ids->ne[0], cur->ne[2], 1 }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ ggml_set_op_params_i32(result, 0, (int32_t) glu_op); ++ ++ result->op = GGML_OP_MOE_FUSED_UP_GATE; ++ result->src[0] = up_exps; ++ result->src[1] = gate_exps; ++ result->src[2] = cur; ++ result->src[3] = ids; ++ ++ return result; ++} ++ + // ggml_out_prod + + static inline bool ggml_can_out_prod(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -362,6 +362,9 @@ extern "C" { + // 0 = off (default), 1 = on, 2 = auto. Effective only when the + // M2 head split is active for a given layer. + uint32_t neo_pipeline_mode; ++ // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md ++ // true = fuse decode MoE up+gate+GLU into one op. false = off (default). ++ bool fused_moe_up_gate; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -66,6 +66,8 @@ llama_context::llama_context( + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + cparams.neo_pipeline_mode = params.neo_pipeline_mode; ++ // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md ++ cparams.fused_moe_up_gate = params.fused_moe_up_gate; + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -3359,6 +3361,7 @@ llama_context_params llama_context_default_params() { + /*.slot_shrink_idle_ms =*/ 0, + /*.headinfer_gpu_heads_frac =*/ 1.0f, + /*.neo_pipeline_mode =*/ 0, // opencoti F5 M3 neo — off by default ++ /*.fused_moe_up_gate =*/ false, // opencoti F5 W3 #292 — off by default + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -32,6 +32,10 @@ struct llama_cparams { + // 2 = auto (same engagement as `on` for now — reserved for future + // load-based engagement heuristics). + uint32_t neo_pipeline_mode; ++ // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md. ++ // true = build_moe_ffn collapses the up/gate expert matmuls + GLU into one ++ // GGML_OP_MOE_FUSED_UP_GATE op at decode (n_tokens==1). false = unfused path. ++ bool fused_moe_up_gate; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -1534,7 +1534,28 @@ ggml_tensor * llm_graph_context::build_moe_ffn( + ggml_tensor * up = nullptr; + ggml_tensor * experts = nullptr; + +- if (gate_up_exps) { ++ // opencoti-hook: f5-opt-fmoe — W3 #292: collapse the separate-up/gate ++ // expert matmuls + the GLU activation into ONE GGML_OP_MOE_FUSED_UP_GATE ++ // op at decode, so the CUDA backend runs the existing fused mmvq kernel ++ // without relying on graph-pattern fusion (ggml_can_fuse_subgraph, which ++ // does not engage for Gemma-4 at decode). Guards mirror supports_op: ++ // decode only (n_tokens==1), separate up/gate of identical type, no ++ // bias/scale, and GELU→GEGLU or SILU→SWIGLU. Off path is byte-identical. ++ bool fused_up_gate = false; ++ if (cparams.fused_moe_up_gate && gate_exps && !gate_up_exps && ++ !up_exps_b && !gate_exps_b && !up_exps_s && !gate_exps_s && ++ up_exps->type == gate_exps->type && n_tokens == 1 && ++ (type_op == LLM_FFN_GELU || type_op == LLM_FFN_SILU)) { ++ const enum ggml_glu_op glu_op = (type_op == LLM_FFN_GELU) ++ ? GGML_GLU_OP_GEGLU : GGML_GLU_OP_SWIGLU; ++ cur = ggml_moe_up_gate(ctx0, up_exps, gate_exps, cur, selected_experts, glu_op); ++ cb(cur, "ffn_moe_fused_up_gate", il); ++ fused_up_gate = true; ++ } ++ ++ if (fused_up_gate) { ++ // cur already holds act(gate) (.) up from the fused op ++ } else if (gate_up_exps) { + // merged gate_up path: one mul_mat_id, then split into gate and up views + ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts); // [n_ff*2, n_expert_used, n_tokens] + cb(gate_up, "ffn_moe_gate_up", il); +@@ -1601,6 +1622,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn( + + const bool has_gate = gate_exps || gate_up_exps; + ++ // opencoti-hook: f5-opt-fmoe — W3 #292: the fused op already applied the ++ // GLU activation; skip the separate activation pass. ++ if (!fused_up_gate) + switch (type_op) { + case LLM_FFN_SILU: + if (gate_exps) { diff --git a/patches/0070-rolling-kv.patch b/patches/0070-rolling-kv.patch new file mode 100644 index 0000000000000000000000000000000000000000..586fca8bf91a4c21a03c3bfa7e14c6a2eec2837d --- /dev/null +++ b/patches/0070-rolling-kv.patch @@ -0,0 +1,4099 @@ +From: opencoti +Subject: [PATCH 0070] F5 M7 Rolling KV — position-windowed KV residency (shipped) + +opencoti F5 M7 "Rolling KV" (#296/#338). Cumulative slot capturing the whole +W4 workstream: auto-residency + runtime tactic table + the GPU streaming op + +the Stage-3 position-window forward + the Stage-4 `--kv-residency-mode` knob. +Provenance: opencoti-original; design in docs/features/rolling_kv.md and +docs/features/advanced_kv.md (M7). Applies after 0042; reverse-applies clean. +The orthogonal cosmocc state-IO fix (bug-269) ships separately as 0071. + +WHAT SHIPS (the position-window tactic, default `--kv-residency-mode auto`): + An iSWA model's overflowing GLOBAL KV layer carries a device-resident window + K/V [0, wc) plus a pinned-host tail [wc, n_kv) merged by an online-softmax + combine. `wc` is sized from the live VRAM budget (`--vram-target`, else all + free VRAM minus a compute reserve); the tactic is auto-selected at cache + construction from the eligibility predicate (offload && !v_trans && + 0=x1 links; kept in-tree for sub-x1 PCIe). ggml-cpu/ops.{cpp,h}, iqk. + * position-axis storage (device window + pinned-host tail tensors), write + routing (cpy_k/cpy_v prefix/suffix split), window/tail accessors, + POSITION_WINDOW tactic + build_rolling_kv_plan + the host-adaptive + compute/bandwidth selector, and state-IO window/tail save+load split: + src/llama-kv-cache{,-iswa}.{cpp,h}. + * build_attn_mha_position_window + iSWA dispatch: src/llama-graph.{cpp,h}. + * --kv-residency-mode / --vram-target / --pcie-bw-gbps threading: + common/{arg,common}.{cpp,h} -> include/llama.h -> src/llama-cparams.h -> + src/llama-context.cpp -> the kv-cache + hybrid + hybrid-iswa ctor sites + (the hybrid sites pass 0 = auto; bug-362). server.cpp pcie write-back. + * op-scoped host-tail copy suppression: ggml/src/ggml-backend.cpp. + +DEFAULT/OFF SAFETY: with KV fitting VRAM the window is empty (single resident +FA, byte-identical to vanilla); `--kv-residency-mode head` runs the M2 split. +All Stage-2 env-gated diagnostics were removed in S4-2 (load-bearing logic +hardcoded to production defaults). Acceptance is logit-distribution +equivalence vs vanilla (.opencoti/rolling-kv-equiv-gate.sh), NOT greedy +byte-equality (tiled online-softmax is fp-non-associative; bug-263 resolved). +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1375,91 +1375,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.slot_initial_ctx = value; + } + ).set_env("LLAMA_ARG_SLOT_INITIAL_CTX").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_PARALLEL})); +- // opencoti F5 M1 rest-kv-eviction — see docs/features/advanced_kv.md +- add_opt(common_arg( +- {"--rest-kv-eviction"}, +- {"--no-rest-kv-eviction"}, +- string_format("when context shift fires, evict the lowest-retention (KeyDiff key-similarity) contiguous window instead of the positional middle (default: %s)", params.rest_kv_eviction ? "enabled" : "disabled"), +- [](common_params & params, bool value) { +- params.rest_kv_eviction = value; +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_EVICTION")); +- add_opt(common_arg( +- {"--rest-kv-recent"}, "N", +- string_format("rest-kv: protect the most-recent N tokens from eviction (default: %d)", params.rest_kv_recent), +- [](common_params & params, int value) { +- params.rest_kv_recent = value; +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_RECENT")); +- add_opt(common_arg( +- {"--rest-kv-layer"}, "N", +- string_format("rest-kv: model layer whose keys score retention, -1 = auto/middle (default: %d)", params.rest_kv_layer), +- [](common_params & params, int value) { +- params.rest_kv_layer = value; +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER")); +- // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md +- add_opt(common_arg( +- {"--headinfer-gpu-heads-frac"}, "F", +- string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac), +- [](common_params & params, const std::string & value) { +- params.headinfer_gpu_heads_frac = std::stof(value); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC")); +- // opencoti F5 M3 neo — see docs/features/advanced_kv.md +- add_opt(common_arg( +- {"--neo-pipeline"}, "MODE", +- "neo (asymmetric GPU/CPU attention pipelining): off (default), on (engage when headinfer split is active for the layer), auto (reserved, same as on). Requires --headinfer-gpu-heads-frac < 1.0 to do anything.", +- [](common_params & params, const std::string & value) { +- if (value == "off") params.neo_pipeline_mode = 0; +- else if (value == "on") params.neo_pipeline_mode = 1; +- else if (value == "auto") params.neo_pipeline_mode = 2; +- else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); +- // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md +- add_opt(common_arg( +- {"--iqk-flash-attn"}, "MODE", +- "iqk CPU flash-attention (vendored ik_llama.cpp engine): off (default), on. " +- "Speeds up the M2 CPU-half attention on the token-generation path; falls back " +- "to the mainline kernel for unsupported (type, shape) cases. x86_64-only.", +- [](common_params & params, const std::string & value) { +- if (value == "off" || value == "false" || value == "0") params.iqk_flash_attn = false; +- else if (value == "on" || value == "true" || value == "1") params.iqk_flash_attn = true; +- else throw std::invalid_argument("--iqk-flash-attn must be on|off"); +- ggml_iqk_flash_attn_set_enabled(params.iqk_flash_attn); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_IQK_FLASH_ATTN")); +- // opencoti F5-opt W3 (#292) fused-MoE — see docs/features/advanced_kv.md +- add_opt(common_arg( +- {"--fused-moe-up-gate"}, "MODE", +- "fused MoE up+gate+GLU (CUDA decode): off (default), on. Collapses the " +- "separate-up/gate expert matmuls + GLU into one op at decode so the CUDA " +- "backend runs the fused mmvq kernel without graph-pattern fusion. Only " +- "engages for separate-up/gate MoE models (e.g. Gemma-4 A4B) at n_tokens==1.", +- [](common_params & params, const std::string & value) { +- if (value == "off" || value == "false" || value == "0") params.fused_moe_up_gate = false; +- else if (value == "on" || value == "true" || value == "1") params.fused_moe_up_gate = true; +- else throw std::invalid_argument("--fused-moe-up-gate must be on|off"); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_FUSED_MOE_UP_GATE")); +- // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md +- add_opt(common_arg( +- {"--pcie-autodetect"}, "MODE", +- "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.", +- [](common_params & params, const std::string & value) { +- if (value == "on" || value == "true" || value == "1") params.pcie_autodetect = true; +- else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false; +- else throw std::invalid_argument("--pcie-autodetect must be on|off"); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT")); +- add_opt(common_arg( +- {"--pcie-bw-gbps"}, "F", +- string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps), +- [](common_params & params, const std::string & value) { +- params.pcie_bw_gbps = std::stof(value); +- } +- ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS")); + add_opt(common_arg( + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + {"--slot-shrink-idle-ms"}, "N", +@@ -1511,13 +1426,43 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_REST_KV_LAYER")); + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md ++ // opencoti F5 M7 Rolling KV — Rung 0: accepts the literal "auto" which sets the ++ // sentinel -1, telling the KV-cache constructor to size residency from free VRAM ++ // (GPU_RESIDENT when KV fits, else a fit-sized CPU spill). A numeric value is the ++ // manual escape hatch (the upstream M2 fixed split). See docs/features/rolling_kv.md. + add_opt(common_arg( +- {"--headinfer-gpu-heads-frac"}, "F", +- string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac), ++ {"--headinfer-gpu-heads-frac"}, "F|auto", ++ string_format("headinfer: fraction of KV heads kept GPU-resident; the rest are offloaded to host memory and streamed back per step (default: %.2f, 1.0 = off). \"auto\" sizes the fraction from free VRAM (with --vram-target). Only effective for GPU-offloaded layers with flash-attention.", (double) params.headinfer_gpu_heads_frac), + [](common_params & params, const std::string & value) { +- params.headinfer_gpu_heads_frac = std::stof(value); ++ if (value == "auto") { ++ params.headinfer_gpu_heads_frac = -1.0f; // opencoti M7 Rung 0 auto-residency sentinel ++ } else { ++ params.headinfer_gpu_heads_frac = std::stof(value); ++ } + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HEADINFER_GPU_HEADS_FRAC")); ++ // opencoti F5 M7 Rolling KV — Rung 0 auto residency — see docs/features/rolling_kv.md ++ add_opt(common_arg( ++ {"--vram-target"}, "MiB", ++ string_format("rolling-kv: VRAM budget cap in MiB for auto KV residency (used with --headinfer-gpu-heads-frac auto). 0 = use all free VRAM minus a compute reserve (maximize). (default: %d)", params.vram_target_mib), ++ [](common_params & params, int value) { ++ params.vram_target_mib = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_VRAM_TARGET")); ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob — see docs/features/rolling_kv.md ++ add_opt(common_arg( ++ {"--kv-residency-mode"}, "MODE", ++ "rolling-kv: KV residency tactic when the cache overflows VRAM — auto (default), head, window. " ++ "auto = position-window when the cache is eligible (offloaded, non-transposed-V, append-only " ++ "non-SWA cache with a real overflow), else the M2 head split. head = force the M2 head split " ++ "(legacy). window = force position-window (falls back with a warning if the cache is ineligible).", ++ [](common_params & params, const std::string & value) { ++ if (value == "auto") params.kv_residency_mode = 0; ++ else if (value == "head") params.kv_residency_mode = 1; ++ else if (value == "window") params.kv_residency_mode = 2; ++ else throw std::invalid_argument("--kv-residency-mode must be auto|head|window"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_KV_RESIDENCY_MODE")); + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + add_opt(common_arg( + {"--neo-pipeline"}, "MODE", +@@ -1542,6 +1487,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + ggml_iqk_flash_attn_set_enabled(params.iqk_flash_attn); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_IQK_FLASH_ATTN")); ++ // opencoti F5-opt W3 (#292) fused-MoE — see docs/features/advanced_kv.md ++ add_opt(common_arg( ++ {"--fused-moe-up-gate"}, "MODE", ++ "fused MoE up+gate+GLU (CUDA decode): off (default), on. Collapses the " ++ "separate-up/gate expert matmuls + GLU into one op at decode so the CUDA " ++ "backend runs the fused mmvq kernel without graph-pattern fusion. Only " ++ "engages for separate-up/gate MoE models (e.g. Gemma-4 A4B) at n_tokens==1.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.fused_moe_up_gate = false; ++ else if (value == "on" || value == "true" || value == "1") params.fused_moe_up_gate = true; ++ else throw std::invalid_argument("--fused-moe-up-gate must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_FUSED_MOE_UP_GATE")); + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--pcie-autodetect"}, "MODE", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1634,6 +1634,13 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms < 0 ? 0u : (uint32_t) params.slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; ++ // opencoti F5 M7 Rolling KV — Rung 0 — see docs/features/rolling_kv.md ++ cparams.vram_target_mib = params.vram_target_mib < 0 ? 0u : (uint32_t) params.vram_target_mib; ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304). PCIe bandwidth for ++ // streaming-FA tile sizing, resolved by the boot-time probe (W1/0036). ++ cparams.pcie_bw_gbps = params.pcie_bw_gbps; ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md ++ cparams.kv_residency_mode = params.kv_residency_mode; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + cparams.neo_pipeline_mode = params.neo_pipeline_mode; + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -440,6 +440,16 @@ struct common_params { + // memory and streamed back per step. 1.0 = off (no split). Only active for + // GPU-offloaded layers with flash-attention (non-transposed V cache). + float headinfer_gpu_heads_frac = 1.0f; ++ // opencoti F5 M7 Rolling KV — Rung 0 auto residency — see docs/features/rolling_kv.md ++ // VRAM budget cap (MiB) for auto KV residency (used with ++ // --headinfer-gpu-heads-frac auto). 0 = use all free VRAM minus a compute ++ // reserve (maximize). Negative is clamped to 0 when copied into cparams. ++ int32_t vram_target_mib = 0; ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob — see docs/features/rolling_kv.md ++ // 0 = auto (POSITION_WINDOW when the cache is eligible, else the M2 head split), ++ // 1 = head (force the M2 head split / never window), 2 = window (force POSITION_WINDOW, ++ // fall back with a warning when ineligible). ++ uint32_t kv_residency_mode = 0; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + // 0 = off (default; M2 concat path runs), 1 = on (engage when split + // active for the layer), 2 = auto (reserved for future load-based +@@ -460,37 +470,6 @@ struct common_params { + // x86_64-only at runtime (the engine compiles only on the AVX2 x86_64 slice). + bool iqk_flash_attn = false; + float pcie_bw_gbps = 0.0f; +- // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md +- // Initial KV-cell allocation per sequence. 0 = full n_ctx_seq allocation +- // (Phase 1 behavior — only zero-fill is deferred). Positive values cap +- // the initial allocation and enable grow-on-demand up to n_ctx_seq. +- int32_t slot_initial_ctx = 0; +- // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md +- // Idle threshold (ms) after which a grown soft cap is shrunk back to +- // slot_initial_ctx and the over-cap pages are decommitted via +- // posix_madvise. 0 disables shrink-on-idle (Phase 2 behavior). +- int32_t slot_shrink_idle_ms = 0; +- // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md +- // Fraction of KV heads kept GPU-resident; the rest are offloaded to host +- // memory and streamed back per step. 1.0 = off (no split). Only active for +- // GPU-offloaded layers with flash-attention (non-transposed V cache). +- float headinfer_gpu_heads_frac = 1.0f; +- // opencoti F5 M3 neo — see docs/features/advanced_kv.md +- // 0 = off (default; M2 concat path runs), 1 = on (engage when split +- // active for the layer), 2 = auto (reserved for future load-based +- // engagement; same as `on` today). +- uint32_t neo_pipeline_mode = 0; +- // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md +- // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 +- // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / +- // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default). +- bool pcie_autodetect = true; +- // opencoti F5-opt W2 (#290) iqk CPU flash-attention — see docs/features/advanced_kv.md +- // Engage the vendored ik_llama.cpp CPU Flash-Attention engine on the M2 +- // CPU-half (token-generation path). Off by default; flag-off is byte-identical. +- // x86_64-only at runtime (the engine compiles only on the AVX2 x86_64 slice). +- bool iqk_flash_attn = false; +- float pcie_bw_gbps = 0.0f; + int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) + int32_t n_keep = 0; // number of tokens to keep from initial prompt +diff --git a/llama.cpp/ggml/include/ggml-iqk-flash-attn.h b/llama.cpp/ggml/include/ggml-iqk-flash-attn.h +--- a/llama.cpp/ggml/include/ggml-iqk-flash-attn.h ++++ b/llama.cpp/ggml/include/ggml-iqk-flash-attn.h +@@ -52,6 +52,21 @@ bool iqk_flash_attn_noalibi(int type_q, int type_mask, float max_bias, + void * work_buffer, barrier_t barrier, void * barrier_data, + int ith, int nth, int n_swa); + ++// opencoti-hook: f5-rolling-kv — W4 M7 S3 (#354). iqk-backed CPU tail flash-attention ++// partial for the CPU_FA_TAIL rolling-KV tactic. Re-declared HERE (not in ++// ggml/src/iqk/iqk_mul_mat.h) so the ggml-cpu tail-partial op (ggml-cpu/ops.cpp) can ++// call it without the ARM-shim header (same reason as iqk_flash_attn_noalibi above). ++// Computes ONE q row's online-softmax over the host-resident tail K/V (nq1=1) and writes ++// the UN-normalized [M, S, VKQ] partial the CUDA streaming combine consumes (O=VKQ/S, ++// lse=M+logS) — byte-compatible with ops.cpp _f16_one_chunk write_partials. The iqk ++// engine uses f32 internal accumulation + the native Dk=Dv=512 blocked kernel (closer to ++// the GPU MMA than the scalar _f16_one_chunk, which ceilings at niah~80). Returns false on ++// unsupported (type, shape); the caller then falls back to the mainline _f16_one_chunk. ++bool iqk_flash_attn_tail_partial(int type_k, int type_v, int Dk, int Dv, ++ int nk1, int stride_k, int stride_v, int stride_m, ++ const float * q, const void * k, const void * v, const void * mask, ++ float scale, float softcap, float * partial); ++ + #ifdef __cplusplus + } + #endif +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -585,6 +585,10 @@ extern "C" { + + GGML_OP_MOE_FUSED_UP_GATE, // opencoti-hook: f5-opt-fmoe — W3 #292 + ++ GGML_OP_STREAMING_FLASH_ATTN, // opencoti-hook: f5-rolling-kv — W4 M7-D #304 ++ ++ GGML_OP_FLASH_ATTN_TAIL_PARTIAL, // opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353 ++ + GGML_OP_COUNT, + }; + +@@ -2423,6 +2427,56 @@ extern "C" { + float max_bias, + float logit_softcap); + ++ // opencoti F5 M7 Rolling KV (W4 M7-D #304) — streaming flash attention. ++ // Same contract as ggml_flash_attn_ext (q,k,v,mask,scale,max_bias, ++ // logit_softcap → F32 [n_embd_v, n_head_q, n_q, n_batch]) but the CUDA ++ // forward tiles over the key dimension and combines tiles with a ++ // cross-tile online-softmax (running max + denom in F32) so the full K/V ++ // need not fit one resident slot. CUDA-only (CPU forward aborts). The ++ // tile/slot sizing is decided inside the op from device free-VRAM; the ++ // graph builder passes cparams.pcie_bw_gbps via op_params for the ++ // double-buffer (R2-b). At R2-a the source K/V may still be GPU-resident ++ // — the op's only job is to prove tiled online-softmax == monolithic FA. ++ GGML_API struct ggml_tensor * ggml_streaming_flash_attn( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap); ++ ++ // opencoti F5 M7 Stage 3c-5 (#341) — position-window streaming FA: resident ++ // device window K/V (k_win/v_win) + pinned-host overflow tail K/V ++ // (k_tail/v_tail, may be NULL ⇒ single resident region). One online-softmax ++ // combine over both; mask spans the full key range [0, n_win + n_tail). ++ GGML_API struct ggml_tensor * ggml_streaming_flash_attn_window( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k_win, ++ struct ggml_tensor * v_win, ++ struct ggml_tensor * k_tail, ++ struct ggml_tensor * v_tail, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap); ++ ++ // opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353. CPU tail-partial emitter for ++ // the CPU_FA_TAIL tactic. dst = [2+DV, n_head, n_q, n_b] holding [M, S, VKQ] ++ // (un-normalized) per output row — the partial the CUDA online-softmax combine ++ // consumes (O=VKQ/S, lse=M+log S). See ggml.c for the full contract. ++ GGML_API struct ggml_tensor * ggml_flash_attn_ext_tail_partial( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap); ++ + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); +diff --git a/llama.cpp/ggml/src/ggml-backend.cpp b/llama.cpp/ggml/src/ggml-backend.cpp +--- a/llama.cpp/ggml/src/ggml-backend.cpp ++++ b/llama.cpp/ggml/src/ggml-backend.cpp +@@ -1247,6 +1247,23 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra + for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) { + ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id); + } ++ // opencoti-hook: f5-rolling-kv — bug-259. STREAMING_FLASH_ATTN (and the ++ // MOE_FUSED_UP_GATE sibling) are CUDA-only ops with no CPU kernel. CUDA's ++ // supports_op gates on get_best_fattn_kernel, which returns NONE for the ++ // tiny -fit / empty-warmup TRIAL graphs — so the op is unassigned here and ++ // (with CPU now reporting it unsupported too) would hit the assert below. ++ // Pin it to the highest-priority GPU backend (CPU is the last index by this ++ // scheduler's own convention, lines ~1025). The -fit pass only reserves ++ // memory (no op execution) and the empty warmup run executes zero rows, so ++ // a conservative trial-graph kernel check is harmless; the real inference ++ // graph is assigned to CUDA normally (FA is supported there). The op's ++ // CUDA forward self-selects the per-tile kernel + resident fallback, and the ++ // S3c copy-suppression (~:1285) already handles its pinned-host K/V srcs. ++ if (*cur_backend_id == -1 && sched->n_backends > 1 && ++ (node->op == GGML_OP_STREAMING_FLASH_ATTN || node->op == GGML_OP_MOE_FUSED_UP_GATE)) { ++ *cur_backend_id = 0; ++ SET_CAUSE(node, "4.cuda-only"); ++ } + GGML_ASSERT(*cur_backend_id != -1); + } + +@@ -1357,7 +1374,31 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra + } + } + +- if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) { ++ // opencoti-hook: f5-rolling-kv — S3c scheduler bypass. ++ // GGML_OP_STREAMING_FLASH_ATTN reads its CPU-half K/V (src 1,2) ++ // directly from the pinned (cuda_host) buffer and DMAs tiles ++ // host->device itself on its copy_stream (fattn.cu). Suppress the ++ // scheduler's full-CPU-half device materialization for THIS op ++ // only; node->src[j] is left pointing at the host view so the cpy ++ // store->read edge on k_cpu_per_stream survives (correctness) and ++ // the op recovers the pinned-host pointer via src->data. Scoped to ++ // the distinct op type → inert for every other op. ++ // The host streaming path is the product default: the op reads its ++ // host-resident K/V in place and DMAs tiles itself, so suppress the ++ // scheduler's device materialization for any host-buffer src on this op. ++ // 3c-5: the POSITION_WINDOW variant carries the pinned-host overflow TAIL ++ // at src[5]/src[6] (the device window is src[1]/src[2]); the host-buffer ++ // guard leaves the device window untouched, so the single-region path ++ // (no src[5]) is byte-identical. ++ bool s3c_bypass = false; ++ if (node->op == GGML_OP_STREAMING_FLASH_ATTN && (j == 1 || j == 2 || j == 5 || j == 6)) { ++ ggml_backend_buffer_t sbuf = src->view_src ? src->view_src->buffer : src->buffer; ++ if (sbuf != NULL && ggml_backend_buffer_is_host(sbuf)) { ++ s3c_bypass = true; ++ } ++ } ++ ++ if (src_backend_id != cur_backend_id && !s3c_bypass && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) { + // create a copy of the input in the split's backend + if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) { + ggml_backend_t backend = sched->backends[cur_backend_id]; +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -2035,6 +2035,12 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + { + ggml_compute_forward_flash_attn_ext(params, tensor); + } break; ++ case GGML_OP_FLASH_ATTN_TAIL_PARTIAL: ++ { ++ // opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353. CPU tail-partial ++ // emitter for the CPU_FA_TAIL tactic (host-resident tail K/V). ++ ggml_compute_forward_flash_attn_ext_tail_partial(params, tensor); ++ } break; + case GGML_OP_FLASH_ATTN_BACK: + { + int32_t t = ggml_get_op_params_i32(tensor, 0); +@@ -2073,6 +2079,16 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + // owns the FFN nodes, so the CPU scheduler never reaches here. + GGML_ABORT("MOE_FUSED_UP_GATE has no CPU implementation (CUDA-only op)"); + } break; ++ case GGML_OP_STREAMING_FLASH_ATTN: ++ { ++ // opencoti-hook: f5-rolling-kv — W4 M7-D #304. CUDA-only op. The ++ // build_attn_mha_streaming graph hook emits it only when the ++ // GPU_STREAM tactic is assigned (which requires the CUDA ++ // backend), so the CPU scheduler never reaches here. The CPU ++ // tactic for an over-budget layer is CPU_SPILL (the W2 iqk FA ++ // engine), a different path entirely. ++ GGML_ABORT("STREAMING_FLASH_ATTN has no CPU implementation (CUDA-only op)"); ++ } break; + case GGML_OP_GET_REL_POS: + { + ggml_compute_forward_get_rel_pos(params, tensor); +@@ -2346,6 +2362,12 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + // scheduled on CPU. Single task keeps the planner well-formed. + n_tasks = 1; + } break; ++ case GGML_OP_STREAMING_FLASH_ATTN: ++ { ++ // opencoti-hook: f5-rolling-kv — W4 M7-D #304. CUDA-only; never ++ // scheduled on CPU. Single task keeps the planner well-formed. ++ n_tasks = 1; ++ } break; + case GGML_OP_SILU_BACK: + case GGML_OP_MUL: + case GGML_OP_DIV: +@@ -2423,6 +2445,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + case GGML_OP_ARGSORT: + case GGML_OP_TOP_K: + case GGML_OP_FLASH_ATTN_EXT: ++ case GGML_OP_FLASH_ATTN_TAIL_PARTIAL: // opencoti-hook: f5-rolling-kv — S3-a #353 + case GGML_OP_FLASH_ATTN_BACK: + case GGML_OP_SSM_CONV: + case GGML_OP_SSM_SCAN: +@@ -3002,6 +3025,17 @@ struct ggml_cplan ggml_graph_plan( + cur = MAX(cur, iqk_fa_work_buffer_size(node, n_tasks)); + #endif + } break; ++ case GGML_OP_FLASH_ATTN_TAIL_PARTIAL: ++ { ++ // opencoti-hook: f5-rolling-kv — S3-a #353. Per-thread ++ // _f16_one_chunk scratch only (the [M,S,VKQ] partial lands in ++ // dst, not wdata). Mirrors that fn's per-thread wdata stride ++ // (DK + 2*DV + cache-line pad); +64 floats covers the pad ++ // generously across cache-line sizes. ++ const int64_t DK = node->src[1]->ne[0]; ++ const int64_t DV = node->src[2]->ne[0]; ++ cur += sizeof(float)*n_tasks*(DK + 2*DV + 64); ++ } break; + case GGML_OP_FLASH_ATTN_BACK: + { + const int64_t D = node->src[0]->ne[0]; +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +@@ -468,6 +468,17 @@ static bool GGML_CALL ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev + case GGML_OP_OUT_PROD: + return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && + src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; ++ // opencoti-hook: f5-rolling-kv / f5-opt-fmoe — bug-259. These are CUDA-only ++ // ops (no CPU forward; the dispatch aborts at ggml-cpu.c:2026/2036). Report ++ // them UNSUPPORTED on CPU so the scheduler always pins them to the CUDA ++ // backend. Without this they fall to `default: return true`, and the ggml ++ // offload heuristic can drift the op to CPU for a small/empty (warmup) batch ++ // whose K/V srcs are pinned CUDA_Host buffers → abort. (MOE_FUSED was saved ++ // only by graph-hook placement; STREAMING_FLASH_ATTN under M7 stream-by- ++ // default is not — both fixed here for robustness.) ++ case GGML_OP_STREAMING_FLASH_ATTN: ++ case GGML_OP_MOE_FUSED_UP_GATE: ++ return false; + default: + return true; + } +diff --git a/llama.cpp/ggml/src/ggml-cpu/ops.cpp b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp ++++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp +@@ -8383,7 +8383,12 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( + const int64_t DV = nev0; + const int64_t N = neq1; + +- GGML_ASSERT(ne0 == DV); ++ // opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353. In write_partials mode the ++ // output goes to `partials` (caller-owned), not dst->data, so dst's ne0 is the ++ // caller's layout: the stock FA split_kv_path keeps ne0==DV, while the tail- ++ // partial op uses ne0==2+DV ([M,S,VKQ]). Skip the ne0==DV dst-shape check when ++ // writing partials; the normalized-output path (FLASH_ATTN_EXT) is unchanged. ++ GGML_ASSERT(write_partials || ne0 == DV); + GGML_ASSERT(ne2 == N); + + // input tensor rows must be contiguous +@@ -9227,6 +9232,54 @@ void ggml_compute_forward_flash_attn_ext( + } + } + ++// opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353. CPU forward for ++// GGML_OP_FLASH_ATTN_TAIL_PARTIAL. One online-softmax pass over the FULL (host- ++// resident) tail K/V per output row, writing the un-normalized [M, S, VKQ] partial ++// into dst (shape [2+DV, n_head, n_q, n_b]). Reuses the proven _f16_one_chunk in ++// its write_partials mode (ic over [0, nek1), partial_stride = 2+DV); rows are ++// partitioned across threads. No reduce/normalize here — the partial IS the output; ++// the CUDA online-softmax combine normalizes (O=VKQ/S) and merges with the device ++// window leg. Decode consumer (n_q==1): one_chunk's ir order is head-fastest then, ++// matching the combine's row index. f16/f32 K/V only (mainline path; no iqk fast ++// path — correctness first, the selector measures whatever this costs). ++void ggml_compute_forward_flash_attn_ext_tail_partial( ++ const ggml_compute_params * params, ++ ggml_tensor * dst) { ++ const ggml_tensor * q = dst->src[0]; ++ const ggml_tensor * k = dst->src[1]; ++ const ggml_tensor * v = dst->src[2]; ++ ++ const int64_t DK = k->ne[0]; ++ const int64_t DV = v->ne[0]; ++ const int64_t nek1 = k->ne[1]; ++ ++ GGML_ASSERT(dst->type == GGML_TYPE_F32); ++ GGML_ASSERT(dst->ne[0] == 2 + DV); ++ ++ // rows = (head, q-token, batch) ++ const int64_t nr = q->ne[1] * q->ne[2] * q->ne[3]; ++ ++ const int ith = params->ith; ++ const int nth = params->nth; ++ ++ const int64_t dr = (nr + nth - 1) / nth; ++ const int64_t ir0 = dr * ith; ++ const int64_t ir1 = MIN(ir0 + dr, nr); ++ ++ if (ir0 >= ir1) { ++ return; ++ } ++ ++ // opencoti F5 M7 S3 (#354): CPU tail-partial. Single online-softmax pass over the ++ // host-resident tail K/V, emitting each row's [M, S, VKQ_unnorm] partial exactly where ++ // the streaming online-softmax combine expects it. Reuses the mainline _f16_one_chunk ++ // write_partials mode (zero recompute). Dormant unless the S2 selector picks CPU_FA_TAIL ++ // (sub-x1 PCIe link; gated out of auto-select per #354). ++ ggml_compute_forward_flash_attn_ext_f16_one_chunk( ++ params, dst, (int) ir0, (int) ir1, 0, nek1, ++ (float *) dst->data, 2 + DV); ++} ++ + // ggml_compute_forward_flash_attn_back + + static void ggml_compute_forward_flash_attn_back_f32( +diff --git a/llama.cpp/ggml/src/ggml-cpu/ops.h b/llama.cpp/ggml/src/ggml-cpu/ops.h +--- a/llama.cpp/ggml/src/ggml-cpu/ops.h ++++ b/llama.cpp/ggml/src/ggml-cpu/ops.h +@@ -87,6 +87,8 @@ void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, + void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst); + void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst); + void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst); ++// opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353. Emits the [M,S,VKQ] tail partial. ++void ggml_compute_forward_flash_attn_ext_tail_partial(const struct ggml_compute_params * params, struct ggml_tensor * dst); + void ggml_compute_forward_flash_attn_back( + const struct ggml_compute_params * params, + const bool masked, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -41,6 +41,13 @@ typedef void (* fattn_kernel_t)( + const int32_t ne31, const int32_t ne32, const int32_t ne33, + const int32_t nb31, const int32_t nb32, const int64_t nb33); + ++// opencoti-hook: rolling-kv M7 (Stage 3a) — consume-once side channel for requesting a per-row ++// log-sum-exp output from the next launch_fattn, without threading dst_lse through the 5-level ++// MMA dispatch (mma_f16 → switch_ncols2 → switch_ncols1 → case → launch_fattn). The windowed-FA ++// op sets this immediately before dispatching one FA; launch_fattn drains it (sets it back to ++// nullptr) so it never leaks to an unrelated FA. Defined in fattn.cu. See UPSTREAM_SYNC.md. ++extern thread_local float * opencoti_fattn_dst_lse; ++ + typedef float (*vec_dot_KQ_t)( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds); + +@@ -679,6 +686,7 @@ template // D == head size + __launch_bounds__(D, 1) + static __global__ void flash_attn_stream_k_fixup_uniform( + float * __restrict__ dst, ++ float * __restrict__ dst_lse, // opencoti-hook: rolling-kv M7 — optional per-row log-sum-exp out (nullptr = off) + const float2 * __restrict__ dst_fixup, + const int ne01, const int ne02, + const int ne12, const int nblocks_stream_k, +@@ -753,6 +761,12 @@ static __global__ void flash_attn_stream_k_fixup_uniform( + + // Write back final result: + *dst = dst_val / rowsum; ++ ++ // opencoti-hook: rolling-kv M7 — per-row log-sum-exp for windowed-FA online-softmax combine (Stage 3a). See docs/protocols/UPSTREAM_SYNC.md. ++ if (dst_lse && tid == 0) { ++ const int64_t lse_row = int64_t(sequence)*ne02*ne01 + int64_t(jt)*ne02*ncols1 + zt_Q + (j*ne02 + c); ++ dst_lse[lse_row] = max_val + logf(rowsum); ++ } + } + + // General fixup kernel for the case where the number of blocks per tile is not uniform across tiles +@@ -761,6 +775,7 @@ template // D == head size + __launch_bounds__(D, 1) + static __global__ void flash_attn_stream_k_fixup_general( + float * __restrict__ dst, ++ float * __restrict__ dst_lse, // opencoti-hook: rolling-kv M7 — optional per-row log-sum-exp out (nullptr = off) + const float2 * __restrict__ dst_fixup, + const int ne01, const int ne02, + const int gqa_ratio, +@@ -862,6 +877,12 @@ static __global__ void flash_attn_stream_k_fixup_general( + + // Write back final result: + *dst = dst_val / rowsum; ++ ++ // opencoti-hook: rolling-kv M7 — per-row log-sum-exp for windowed-FA online-softmax combine (Stage 3a). See docs/protocols/UPSTREAM_SYNC.md. ++ if (dst_lse && tid == 0) { ++ const int64_t lse_row = int64_t(sequence)*ne02*ne01 + int64_t(jt)*ne02*ncols1 + zt_Q + (j*ne02 + c); ++ dst_lse[lse_row] = max_val + logf(rowsum); ++ } + } + + template // D == head size +@@ -870,6 +891,7 @@ static __global__ void flash_attn_combine_results( + const float * __restrict__ VKQ_parts, + const float2 * __restrict__ VKQ_meta, + float * __restrict__ dst, ++ float * __restrict__ dst_lse, // opencoti-hook: rolling-kv M7 — optional per-row log-sum-exp out (nullptr = off) + const int parallel_blocks) { + ggml_cuda_pdl_lc(); + // Dimension 0: threadIdx.x +@@ -917,15 +939,27 @@ static __global__ void flash_attn_combine_results( + } + + dst[tid] = VKQ_numerator / VKQ_denominator; ++ ++ // opencoti-hook: rolling-kv M7 — per-row log-sum-exp for windowed-FA online-softmax combine (Stage 3a). See docs/protocols/UPSTREAM_SYNC.md. ++ if (dst_lse && tid == 0) { ++ dst_lse[j_dst_unrolled] = kqmax + logf(VKQ_denominator); ++ } + } + + template + void launch_fattn( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, +- const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE ++ const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE, ++ float * dst_lse = nullptr // opencoti-hook: rolling-kv M7 — per-row LSE out (Stage 3a); nullptr = byte-identical legacy path + ) { + constexpr int ncols = ncols1 * ncols2; + ++ // opencoti-hook: rolling-kv M7 (Stage 3a) — drain the consume-once dst_lse side channel. ++ if (dst_lse == nullptr && opencoti_fattn_dst_lse != nullptr) { ++ dst_lse = opencoti_fattn_dst_lse; ++ } ++ opencoti_fattn_dst_lse = nullptr; ++ + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; +@@ -1171,6 +1205,18 @@ void launch_fattn( + ); + CUDA_CHECK(cudaGetLastError()); + ++ // opencoti-hook: rolling-kv M7 (Stage 3a) — dst_lse is only populated by the finalize kernels ++ // (uniform/general fixup, or the parallel-blocks combine). If the dispatch resolves to the ++ // in-kernel single-block write path, no finalize runs and dst_lse would be left garbage; assert ++ // loudly instead of silently corrupting the windowed-FA online-softmax combine. The in-kernel LSE ++ // emit (typedef-threaded MMA epilogue) is a separate prefill-overflow stage; on the decode path ++ // (stream_k always engaged) a fixup always runs. See docs/protocols/UPSTREAM_SYNC.md. ++ if (dst_lse) { ++ const bool fixup_ran = stream_k && (((int)blocks_num.x % ntiles_dst == 0 && (int)blocks_num.x > ntiles_dst) || (ntiles_dst % (int)blocks_num.x != 0)); ++ const bool combine_ran = !stream_k && parallel_blocks > 1; ++ GGML_ASSERT((fixup_ran || combine_ran) && "rolling-kv: dst_lse requested but FA resolved to the in-kernel write path (no finalize kernel); unsupported on this path"); ++ } ++ + if (stream_k) { + if ((int)blocks_num.x % ntiles_dst == 0 && (int)blocks_num.x > ntiles_dst) { + // Optimized fixup: nblocks_stream_k is a multiple of ntiles_dst, launch one block per tile. +@@ -1186,7 +1232,7 @@ void launch_fattn( + + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_stream_k_fixup_uniform, launch_params, +- (float *) KQV->data, dst_tmp_meta.ptr, ++ (float *) KQV->data, dst_lse, dst_tmp_meta.ptr, + Q->ne[1], Q->ne[2], K->ne[2], nblocks_sk, + gqa_ratio, bpt, fd0, fd1, fd2); + } else if (ntiles_dst % blocks_num.x != 0) { +@@ -1203,7 +1249,7 @@ void launch_fattn( + + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_stream_k_fixup_general, launch_params, +- (float *) KQV->data, dst_tmp_meta.ptr, ++ (float *) KQV->data, dst_lse, dst_tmp_meta.ptr, + Q->ne[1], Q->ne[2], gqa_ratio, total_work, + fd_k_j_z_ne12, fd_k_j_z, fd_k_j, fd_k); + } +@@ -1214,7 +1260,7 @@ void launch_fattn( + + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, nbytes_shared_combine, main_stream); + ggml_cuda_kernel_launch(flash_attn_combine_results, launch_params, +- dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks); ++ dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, dst_lse, parallel_blocks); + } + CUDA_CHECK(cudaGetLastError()); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -6,6 +6,11 @@ + #include "fattn-wmma-f16.cuh" + #include "fattn.cuh" + ++// opencoti-hook: rolling-kv M7 (Stage 3a) — definition of the consume-once dst_lse side channel ++// declared in fattn-common.cuh. Set immediately before a single launch_fattn dispatch to request a ++// per-row log-sum-exp output; launch_fattn drains it back to nullptr. See docs/protocols/UPSTREAM_SYNC.md. ++thread_local float * opencoti_fattn_dst_lse = nullptr; ++ + template + static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; +@@ -560,3 +565,841 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst + bool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst) { + return ggml_cuda_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE; + } ++ ++// ============================================================================ ++// opencoti-hook: f5-rolling-kv — W4 M7-D #304 / R2-b S2 (#311). ++// Streaming flash-attention via key-axis tiling + LSE-weighted online-softmax. ++// ++// The host head-group's K/V (dst->src[1], src[2]) are processed along the key ++// axis in tiles. Each tile runs the STOCK ggml_cuda_flash_attn_ext (untouched — ++// the resident path stays byte-identical, every kernel family is covered) over ++// a key-slice, giving that tile's normalised output O_t. A small reduction ++// emits each tile's per-output-row log-sum-exp lse_t. Tiles merge in F32: ++// O = Σ_t O_t·exp(lse_t − M) / Σ_t exp(lse_t − M), M = max_t lse_t ++// which is the exact online-softmax (algebra verified). Single-tile ++// short-circuits to plain FA (= R2-b S1, byte-identical, zero overhead). ++// ++// Tile count is AUTO (sized from n_kv); a single resolved tile over a resident ++// source short-circuits to plain FA. The slot pool (default 2 slots) stages each ++// tile's K/V through a round-robin GPU slot via a ++// SINGLE-STREAM cudaMemcpyAsync(cudaMemcpyDefault) on the op's own ctx.stream() ++// — no dedicated copy_stream and no events. UVA infers H2D (pinned-host spill) ++// or D2D (resident) per pointer. ctx.stream() rides whatever stream the M2/M7 ++// head-split's concurrent_events forked the op onto, so it cannot collide with ++// that machinery (bug-253) and stays CUDA-graph-capturable. Bit-stable vs the ++// resident path. The dedicated copy_stream + copy_done/compute_done events ++// (the actual double-buffer overlap) are S3b's job; S3c adds the host→device ++// streaming that is the memory win. ++// ++// Scoring coverage: scale + additive mask only — max_bias==0 (no ALiBi), ++// logit_softcap==0, no sinks (src[4]==NULL). Otherwise this falls back to ++// resident FA. Gate model qwen2.5-1.5b satisfies all three. ++// ============================================================================ ++ ++// One block per output row (head, q-token, batch); WARP_SIZE threads cooperate ++// on each head_dim dot. Online (max, sumexp) over the tile's keys → lse[row]. ++static __global__ void streaming_lse_kernel( ++ const char * __restrict__ Q, const char * __restrict__ K, ++ const char * __restrict__ mask, float * __restrict__ lse, ++ const float scale, const float logit_softcap, const int tile_kv, const int gqa_ratio, ++ const int head_dim, const int n_head, const int n_q, ++ const int64_t q_nb1, const int64_t q_nb2, const int64_t q_nb3, ++ const int64_t k_nb1, const int64_t k_nb2, const int64_t k_nb3, ++ const int64_t m_nb0, const int64_t m_nb1) { ++ const int row = blockIdx.x; // flattened (h, qi, b) ++ const int h = row % n_head; ++ const int qi = (row / n_head) % n_q; ++ const int b = row / (n_head * n_q); ++ const int hk = h / gqa_ratio; ++ const int lane = threadIdx.x; ++ ++ const float * Qp = (const float *)(Q + (int64_t)qi*q_nb1 + (int64_t)h*q_nb2 + (int64_t)b*q_nb3); ++ const char * Kb = K + (int64_t)hk*k_nb2 + (int64_t)b*k_nb3; ++ const char * Mb = mask ? (mask + (int64_t)qi*m_nb1) : nullptr; ++ ++ float m = -INFINITY; ++ float l = 0.0f; ++ for (int j = 0; j < tile_kv; ++j) { ++ const half * Kj = (const half *)(Kb + (int64_t)j*k_nb1); ++ float part = 0.0f; ++ for (int d = lane; d < head_dim; d += WARP_SIZE) { ++ part += Qp[d] * __half2float(Kj[d]); ++ } ++#pragma unroll ++ for (int off = WARP_SIZE/2; off > 0; off >>= 1) { ++ part += __shfl_xor_sync(0xFFFFFFFF, part, off, WARP_SIZE); ++ } ++ // S3b softcap (#340): mirror plain FA — softcap*tanhf((scale/softcap)*qk) ++ // BEFORE the mask add (fattn-mma-f16.cuh:611-621 then :640), divide-by-zero ++ // guarded. softcap==0 is the byte-identical pre-S3b path. ++ float s = (logit_softcap != 0.0f) ++ ? logit_softcap * tanhf((scale / logit_softcap) * part) ++ : scale * part; ++ if (Mb) { ++ s += __half2float(*(const half *)(Mb + (int64_t)j*m_nb0)); ++ } ++ if (s == -INFINITY) { // fully-masked key — contributes nothing ++ continue; ++ } ++ const float m_new = fmaxf(m, s); ++ l = l * expf(m - m_new) + expf(s - m_new); // m=-inf first hit: expf(-inf)=0 ++ m = m_new; ++ } ++ if (lane == 0) { ++ lse[row] = (l > 0.0f) ? (m + logf(l)) : -INFINITY; ++ } ++} ++ ++// One thread per output element. dst layout [DV, n_head, n_q, n_b]; each tile's ++// O block and the dst share that contiguous layout, so the row index used for ++// lse is exactly elem/DV — identical to streaming_lse_kernel's `row`. ++static __global__ void streaming_combine_kernel( ++ const float * __restrict__ O_tiles, const float * __restrict__ lse_tiles, ++ float * __restrict__ dst, const int n_tiles, const int dv, const int n_rows) { ++ const int64_t total = (int64_t)dv * n_rows; ++ const int64_t gid = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; ++ if (gid >= total) { ++ return; ++ } ++ const int row = (int)(gid / dv); ++ float M = -INFINITY; ++ for (int t = 0; t < n_tiles; ++t) { ++ M = fmaxf(M, lse_tiles[(int64_t)t*n_rows + row]); ++ } ++ float num = 0.0f; ++ float den = 0.0f; ++ for (int t = 0; t < n_tiles; ++t) { ++ const float lse = lse_tiles[(int64_t)t*n_rows + row]; ++ const float w = (lse == -INFINITY || M == -INFINITY) ? 0.0f : expf(lse - M); ++ if (w != 0.0f) { ++ num += O_tiles[(int64_t)t*total + gid] * w; ++ den += w; ++ } ++ } ++ dst[gid] = (den > 0.0f) ? (num / den) : 0.0f; ++} ++ ++// opencoti-hook: f5-rolling-kv — S3-b (#353) CPU_FA_TAIL. Convert a precomputed ++// CPU tail partial into ONE combine tile. The partial is the [2+DV, n_rows] ++// output of ggml_flash_attn_ext_tail_partial: per output row r (contiguous at ++// r*(2+DV)) it holds [M, S, VKQ_unnorm[0..DV)] — un-normalized online-softmax ++// state over the host-resident tail keys. This kernel writes that row's: ++// O_tile[row*DV + d] = VKQ_unnorm[d] / S (normalized O, DV-fastest — the ++// exact streaming_combine_kernel layout) ++// lse_tile[row] = M + logf(S) (the per-row log-sum-exp) ++// so the tail merges through the UNCHANGED streaming_combine_kernel alongside the ++// device window tile(s). Only the tiny partial (2+DV floats/row) ever crossed ++// PCIe — never the tail K/V (the CPU_FA_TAIL win on slow links). One thread per ++// (row, d) output element. S<=0 (fully-masked / empty tail) → O=0, lse=-inf ++// (the combine ignores a -inf tile). ++static __global__ void tail_partial_to_tile_kernel( ++ const float * __restrict__ partial, float * __restrict__ O_tile, ++ float * __restrict__ lse_tile, const int dv, const int n_rows) { ++ const int64_t total = (int64_t)dv * n_rows; ++ const int64_t gid = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; ++ if (gid >= total) { ++ return; ++ } ++ const int row = (int)(gid / dv); ++ const int d = (int)(gid % dv); ++ const float * pr = partial + (int64_t)row * (2 + dv); ++ const float M = pr[0]; ++ const float S = pr[1]; ++ O_tile[gid] = (S > 0.0f) ? (pr[2 + d] / S) : 0.0f; ++ if (d == 0) { ++ lse_tile[row] = (S > 0.0f) ? (M + logf(S)) : -INFINITY; ++ } ++} ++ ++void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ // opencoti F5 M7: the tile count is AUTO — sized from n_kv below (~the default ++ // keys/tile). The single-tile short-circuit to plain FA is deferred below to ++ // AFTER the host detection — a host (bypass) source must still take the slot lift ++ // even at one tile (plain FA would read host memory as device); only a ++ // device/resident single tile is byte-identically plain FA (the S1 anchor). ++ ++ float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; ++ memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); ++ memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); ++ memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); ++ ++ // Unsupported scoring → keep this layer resident (correctness over relief). ++ // opencoti-hook: f5-rolling-kv — S3b (#340) softcap relief. Plain FA handles ++ // softcap natively, so drop the softcap bail for head_dim>256 (MMA-only: VEC is ++ // excluded at >256, so the per-tile FA always reaches a finalize kernel — see ++ // the dst_lse decode channel below). head_dim<=256 keeps the softcap bail (VEC ++ // could be selected → no finalize → the dst_lse safety assert). ALiBi (max_bias) ++ // and sinks (src[4]) bails stay unconditional (the combine folds neither yet). ++ // dst->src[1]==K, but K is not fetched until below, so read ne[0] off src[1]. ++ if (max_bias != 0.0f || dst->src[4] != nullptr || ++ (logit_softcap != 0.0f && dst->src[1]->ne[0] <= 256)) { ++ ggml_cuda_flash_attn_ext(ctx, dst); ++ return; ++ } ++ ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ // opencoti-hook: f5-rolling-kv — Stage 3c-5 (#346) two-region (position-window) ++ // forward. When src[5]/src[6] (k_tail/v_tail) are present the op fuses a ++ // device-resident WINDOW [0,n_win) (src[1]/src[2], the bulk) and a pinned-host ++ // overflow TAIL [n_win,n_kv) (src[5]/src[6], local idx = global−n_win) into ONE ++ // online-softmax combine: window tiles read the device source, tail tiles read ++ // the host source, every partial merges through the unchanged ++ // streaming_combine_kernel. src[5]==null ⇒ single region (the established ++ // S2/S3a/S3b path), byte-identical. The window and tail share K/V type, head_dim ++ // and n_head_kv (asserted below); only the key-position count (ne[1]) differs, so ++ // the resolve() mapping below treats them as one logical [0,n_kv) key space split ++ // at n_win. ++ // opencoti-hook: f5-rolling-kv — S3-b (#353) CPU_FA_TAIL. op_params[4]==1 marks ++ // "tail is a precomputed CPU partial" mode: src[5] is NOT k_tail but the ++ // [2+DV, n_head, n_q, n_b] partial [M,S,VKQ_unnorm] produced on the CPU by ++ // ggml_flash_attn_ext_tail_partial. In this mode the op runs ordinary ++ // window-only tiling over the device-resident bulk [0,n_win) (src[1]/src[2]) ++ // and merges the converted partial (O=VKQ/S, lse=M+logS) as ONE extra combine ++ // tile — only the tiny partial crossed PCIe, never the tail K/V. src[6] unused. ++ // op_params is zero-initialised (ggml.c new_tensor), so every legacy/two-region ++ // op has op_params[4]==0 ⇒ this branch never fires for them. ++ const bool tail_is_partial = (ggml_get_op_params_i32(dst, 4) == 1); ++ const ggml_tensor * tail_partial = tail_is_partial ? dst->src[5] : nullptr; ++ const ggml_tensor * K_tail = tail_is_partial ? nullptr : dst->src[5]; ++ const ggml_tensor * V_tail = tail_is_partial ? nullptr : dst->src[6]; ++ const bool two_region = (K_tail != nullptr); ++ if (two_region) { ++ GGML_ASSERT(V_tail != nullptr); ++ GGML_ASSERT(K_tail->type == K->type && V_tail->type == V->type); ++ GGML_ASSERT(K_tail->ne[0] == K->ne[0] && V_tail->ne[0] == V->ne[0]); ++ GGML_ASSERT(K_tail->ne[2] == K->ne[2]); ++ } ++ ++ // opencoti-hook: f5-rolling-kv — S3d dequant-on-lift. The streaming slot is ++ // ALWAYS f16 (streaming_lse_kernel reads it via __half2float); a quantized ++ // K/V is dequantized into the slot on the lift using ggml_get_to_fp16_nc_cuda ++ // — the exact primitive fattn-common.cuh:965-1025 uses to convert strided K/V ++ // to f16 (strides in elements = nb/type_size, packed f16 output). This ++ // REPLACES the bug-254 f16 stop-gap and unblocks bug-226 (q8_0 @ 262144). ++ // Two safety guards: (1) an unsupported KV type (no nc converter) stays ++ // resident; (2) a quantized type that ends up with no slot to dequant into ++ // (n_slots==0) stays resident — both enforced below. The f16 path resolves ++ // to nullptr converters and is byte-identical to pre-S3d. ++ const bool all_f16 = (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16); ++ to_fp16_nc_cuda_t to_fp16_k = all_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(K->type); ++ to_fp16_nc_cuda_t to_fp16_v = all_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(V->type); ++ if (!all_f16 && (to_fp16_k == nullptr || to_fp16_v == nullptr)) { ++ ggml_cuda_flash_attn_ext(ctx, dst); // unsupported quant type → resident ++ return; ++ } ++ ++ const int head_dim = K->ne[0]; ++ // 3c-5: n_kv is the TOTAL logical key span — window (src[1]) + tail (src[5]). ++ // Single region keeps n_kv == K->ne[1] (n_tail==0) → byte-identical downstream. ++ const int n_win = K->ne[1]; ++ const int n_tail = two_region ? (int) K_tail->ne[1] : 0; ++ const int n_kv = n_win + n_tail; ++ const int n_head_kv = K->ne[2]; ++ const int n_head = Q->ne[2]; ++ const int n_q = Q->ne[1]; ++ const int n_b = Q->ne[3]; ++ const int gqa = n_head / n_head_kv; ++ const int dv = V->ne[0]; ++ const int n_rows = n_head * n_q * n_b; // == dst->ne[1]*ne[2]*ne[3] ++ ++ // Tile boundaries aligned to FATTN_KQ_STRIDE so every slice is a valid FA ++ // key length; the final tile carries the remainder. ++ const int n_blk = (n_kv + FATTN_KQ_STRIDE - 1) / FATTN_KQ_STRIDE; ++ // Resolve the auto tile count. Target ~DEFAULT_TILE_KEYS keys per tile so a slot ++ // holds far less than the full KV (the streaming memory win). Clamp to [1, n_blk]. ++ // ++ // opencoti-hook: f5-rolling-kv — M7-P1 (#335) launch-slack kill. At 2048 ++ // keys/tile a 256k spill becomes 128 tiles, each firing converter×2 + ++ // per-tile FA + lse (+ one final combine) ≈ hundreds of kernel launches per ++ // token — pure overhead, since total attention work is O(n_kv) regardless of ++ // how it is tiled. Coarsen to 32768 keys/tile so n_tiles at 256k drops ++ // 128 → ~8: the per-tile FA/lse simply process a longer key slice (no extra ++ // compute), the combine reduces ~8 partials instead of 128, and the ++ // inter-tile online-softmax stays exact (associative reduction; cosine- and ++ // RULER-gated). The win is launch overhead only. A VRAM cap keeps the f16 ++ // double-buffer slot pool bounded: each streamed key costs ++ // (head_dim+dv)·sizeof(half)·n_head_kv·n_b across K+V, so a coarse tile that ++ // would push the worst-case slot pool past the slot-pool cap (512 MiB) raises ++ // n_tiles back up — coarsening never OOMs. ++ const int DEFAULT_TILE_KEYS = 32768; ++ int n_tiles = (n_kv + DEFAULT_TILE_KEYS - 1) / DEFAULT_TILE_KEYS; ++ // VRAM clamp so a too-coarse tile cannot OOM. per_key_slot = f16 K+V bytes a ++ // single streamed key occupies in one slot; the pool is n_slots (≤4 below) ++ // copies of tile_kv_full keys. ++ { ++ const size_t per_key_slot = ++ ((size_t)head_dim + (size_t)dv) * sizeof(half) * (size_t)n_head_kv * (size_t)n_b; ++ const size_t cap_mib = 512; ++ const size_t denom = per_key_slot * 4; // worst-case n_slots (S3a cap) ++ size_t mtk = (denom == 0) ? (size_t)n_kv : (cap_mib * 1024 * 1024) / denom; ++ if (mtk < (size_t)FATTN_KQ_STRIDE) mtk = (size_t)FATTN_KQ_STRIDE; ++ const int n_tiles_vram = (n_kv + (int)mtk - 1) / (int)mtk; ++ if (n_tiles < n_tiles_vram) n_tiles = n_tiles_vram; ++ } ++ if (n_tiles < 1) n_tiles = 1; ++ if (n_tiles > n_blk) n_tiles = n_blk; ++ const int blk_per_tile = (n_blk + n_tiles - 1) / n_tiles; ++ const int tile_kv_full = blk_per_tile * FATTN_KQ_STRIDE; ++ ++ // 3c-5: re-tile the WINDOW and TAIL SEPARATELY at the resolved tile_kv_full so no ++ // tile straddles the n_win boundary (a straddle tile would need both a device and ++ // a host source). n_tiles grows by ≤1 partial vs the unified count; the pools and ++ // combine below size for the updated n_tiles. resolve(t) maps t O_tiles (pool, (size_t)n_combine_tiles * nel); ++ ggml_cuda_pool_alloc lse_tiles(pool, (size_t)n_combine_tiles * n_rows); ++ ++ // Slot-pool staging: each tile's K/V is routed through a round-robin GPU slot, ++ // enabling the copy/compute double buffer below. A 2-slot double buffer is the ++ // product default. Slot lift requires a contiguous resident K/V; otherwise the ++ // op falls back to the in-place view path. The lse kernel reads K as half, so ++ // the slot path assumes f16 K. ++ int n_slots = 2; ++ // 3c-5: the two-region forward REQUIRES the double-buffer (window tiles on the ++ // device source interleave with host-tail DMA) — never honour a slot-pool ++ // disable for it. ++ if (two_region && n_slots < 2) n_slots = 2; ++ // opencoti-hook: f5-rolling-kv — S3c scheduler-bypass host streaming. The ++ // scheduler's device copy is suppressed for this op (ggml-backend.cpp), so its ++ // K/V point STRAIGHT at the pinned-host CPU-half: a ++ // permuted (non-contiguous) view whose key rows are strided by ++ // nb[1]=n_head_cpu*row(head_dim), NOT row(head_dim). The packed 1-D lift ++ // below assumes contiguous key rows (the device-dup layout), so a host source ++ // MUST be lifted with a strided cudaMemcpy2DAsync (src pitch = nb[1]); and the ++ // slot path MUST be forced for it (the S2 in-place path would hand a host ++ // pointer to the FA kernel as device → wrong). Device sources keep the exact ++ // S3a/S3b 1-D lift → byte-identical. Detect the host source by pointer type. ++ bool host_src = false; ++ { ++ cudaPointerAttributes kattr = {}; ++ cudaPointerAttributes vattr = {}; ++ const cudaError_t ek = cudaPointerGetAttributes(&kattr, K->data); ++ const cudaError_t ev = cudaPointerGetAttributes(&vattr, V->data); ++ if (ek == cudaSuccess && ev == cudaSuccess && ++ kattr.type == cudaMemoryTypeHost && vattr.type == cudaMemoryTypeHost) { ++ host_src = true; ++ } ++ cudaGetLastError(); // clear any sticky error from probing an unregistered ptr ++ } ++ ++ // 3c-5: probe the TAIL source independently (the window is device-resident → ++ // host_src false; the overflow tail is pinned-host → host_tail true). resolve(t) ++ // hands each path its own host flag so window tiles take the 1-D device lift and ++ // tail tiles take the 2-D pinned-host lift. ++ bool host_tail = false; ++ if (two_region) { ++ cudaPointerAttributes ta = {}; ++ if (cudaPointerGetAttributes(&ta, K_tail->data) == cudaSuccess && ++ ta.type == cudaMemoryTypeHost) { ++ host_tail = true; ++ } ++ cudaGetLastError(); ++ } ++ ++ // 3c-5: per-tile region descriptor. Single region returns exactly the legacy ++ // (kv0 = t·tile_kv_full) mapping → byte-identical. Two region: tiles [0,n_win_tiles) ++ // cover the device window at local==global offset; tiles [n_win_tiles,n_tiles) ++ // cover the host tail at local = (t−n_win_tiles)·tile_kv_full, global = n_win+local. ++ // The mask is sliced by the GLOBAL offset; the source is read at the LOCAL offset. ++ struct tile_region_t { ++ const ggml_tensor * K; ++ const ggml_tensor * V; ++ bool host; ++ int local_kv0; ++ int global_kv0; ++ int this_kv; ++ }; ++ auto resolve = [&](int t) -> tile_region_t { ++ if (!two_region) { ++ const int kv0 = t * tile_kv_full; ++ const int tk = kv0 >= n_kv ? 0 : min(tile_kv_full, n_kv - kv0); ++ return { K, V, host_src, kv0, kv0, tk }; ++ } ++ if (t < n_win_tiles) { ++ const int loc = t * tile_kv_full; ++ const int tk = loc >= n_win ? 0 : min(tile_kv_full, n_win - loc); ++ return { K, V, host_src, loc, loc, tk }; ++ } ++ const int j = t - n_win_tiles; ++ const int loc = j * tile_kv_full; ++ const int tk = loc >= n_tail ? 0 : min(tile_kv_full, n_tail - loc); ++ return { K_tail, V_tail, host_tail, loc, n_win + loc, tk }; ++ }; ++ ++ // A single resolved tile over a device/resident source is exactly ++ // plain FA — the byte-identical S1 anchor, and the correct resident path for a ++ // quantized source too. A host (bypass) source must take the slot lift even at ++ // one tile (plain FA would read host memory as device), so guard on !host_src. ++ // S3-b (#353): never short-circuit in CPU_FA_TAIL mode — a lone window tile is ++ // plain FA over the BULK only; the appended partial tile (the overflow) must ++ // still be converted and combined. Fall through to the tiling+combine path. ++ if (n_tiles == 1 && !host_src && !tail_is_partial) { ++ ggml_cuda_flash_attn_ext(ctx, dst); ++ return; ++ } ++ ++ // S3d: a quantized K/V MUST go through a slot (the S2 in-place path and the ++ // f16 lse kernel read K->data as f16 → wrong for quant), so !all_f16 forces ++ // the slot path; with no slot to dequant into it falls back to resident. ++ const bool use_slots = n_slots > 0 && ++ (two_region || host_src || !all_f16 || (ggml_is_contiguous(K) && ggml_is_contiguous(V))); ++ if (!all_f16 && !use_slots) { ++ ggml_cuda_flash_attn_ext(ctx, dst); // quant but no slot → resident ++ return; ++ } ++ ++ if (use_slots) { ++ if (n_slots > n_tiles) n_slots = n_tiles; ++ if (n_slots > 4) n_slots = 4; // S3a pool cap ++ ++ // S3d: the slot is ALWAYS f16 (dequant-on-lift), so it is sized for f16 ++ // regardless of K/V storage type. For f16 source this equals the prior ++ // ggml_row_size(F16,·) → byte-identical anchor preserved. ++ const size_t k_row = (size_t)head_dim * sizeof(half); ++ const size_t v_row = (size_t)dv * sizeof(half); ++ const size_t k_slot = k_row * (size_t)tile_kv_full * n_head_kv * n_b; ++ const size_t v_slot = v_row * (size_t)tile_kv_full * n_head_kv * n_b; ++ // S3d: type-size of the SOURCE (storage) type → nc-converter strides are ++ // s = nb/ts (blocks-per-row for quant, elements for f16). Unused on the ++ // all_f16 fast path (the memcpy lift) but cheap to compute. ++ const size_t ts_k = ggml_type_size(K->type); ++ const size_t ts_v = ggml_type_size(V->type); ++ ggml_cuda_pool_alloc slotK(pool, (size_t)n_slots * k_slot); ++ ggml_cuda_pool_alloc slotV(pool, (size_t)n_slots * v_slot); ++ ++ // R2-b S3b (#312): with >=2 slots the op runs a double-buffer ping-pong — the ++ // per-tile lift is issued on a dedicated copy_stream while FA/lse for the ++ // current tile run on ctx.stream(), so tile t+1's DMA overlaps tile t's ++ // compute. copy_done/compute_done events (pre-created, record/wait only) order ++ // the two streams; we never cudaEventCreate inside the op so the path stays ++ // cuda_graph-capturable (bug-251 was event *creation* during capture). ++ // copy_stream is the fixed top index (GGML_CUDA_MAX_STREAMS-1), clear of the ++ // QKV concurrent-event fan-out (streams 1-3, which join before this op) and ++ // NEO's stream 1 — the collision that was bug-253. <2 slots → S3a single-stream. ++ // 3c-5: two-region always runs the overlap path (the S3a/S2 single-stream ++ // branches use the uniform kv0=t·tile_kv_full mapping, which would straddle ++ // the window/tail boundary). n_slots≥2 is already forced above. ++ const bool use_overlap = two_region || (n_slots >= 2); ++ ++ if (use_overlap) { ++ const int cs_idx = GGML_CUDA_MAX_STREAMS - 1; // fixed, collision-free ++ GGML_ASSERT(cs_idx >= 1); ++ cudaStream_t cs = ctx.stream(ctx.device, cs_idx); ++ ++ // Pre-created event pool (>= S3a slot cap 4). cudaEventDisableTiming; ++ // created ONCE during the eager warmup (op dispatch is single-thread, ++ // the first eval predates capture) → never a capture-time cudaEventCreate. ++ static cudaEvent_t copy_done[4] = {}; ++ static cudaEvent_t compute_done[4] = {}; ++ static bool ev_init = false; ++ if (!ev_init) { ++ for (int s = 0; s < 4; ++s) { ++ CUDA_CHECK(cudaEventCreateWithFlags(©_done[s], cudaEventDisableTiming)); ++ CUDA_CHECK(cudaEventCreateWithFlags(&compute_done[s], cudaEventDisableTiming)); ++ } ++ ev_init = true; ++ } ++ ++ // bug-255: the slot pool buffers are reclaimed/re-handed across op ++ // invocations (every layer reuses the same pool addresses). In S3a ++ // copy+FA shared one stream so layer L+1's copy was implicitly ordered ++ // after layer L's FA. With the copy moved to copy_stream, layer L+1's ++ // lift could overwrite a slot while layer L's FA (on the compute ++ // stream) is still reading it → cross-op WAR. Order copy_stream after ++ // all prior compute-stream work so a reused slot is never clobbered ++ // mid-read. (One event per op entry; intra-op order is the ping-pong.) ++ static cudaEvent_t cs_sync = nullptr; ++ if (cs_sync == nullptr) { ++ CUDA_CHECK(cudaEventCreateWithFlags(&cs_sync, cudaEventDisableTiming)); ++ } ++ CUDA_CHECK(cudaEventRecord(cs_sync, stream)); ++ CUDA_CHECK(cudaStreamWaitEvent(cs, cs_sync, 0)); ++ ++ // Lift tile t's K/V into its slot on the copy_stream (cudaMemcpyDefault ++ // → UVA H2D/D2D). Same packed 1-D-per-(b,h) layout as the S3a path. ++ auto lift = [&](int t) { ++ const tile_region_t tr = resolve(t); ++ const int this_kv = tr.this_kv; ++ if (this_kv <= 0) return; ++ const ggml_tensor * tK = tr.K; // 3c-5: window (device) | tail (host) ++ const ggml_tensor * tV = tr.V; ++ const int loc = tr.local_kv0; // region-local source offset ++ // 3c-5: the all-f16 lift packs head hh's keys into a contiguous slot. ++ // The packed 1-D copy is valid ONLY when the source already has each ++ // head's keys contiguous (nb[1]==row). The position-window device view ++ // (permuted, ALL heads per cell → nb[1]==n_embd_gqa*ts != row) is NOT, ++ // so it MUST take the 2-D strided lift like the host tail. Branch on ++ // contiguity for two_region; single region keeps the exact host_src ++ // predicate (byte-identical: there a host src is always the strided ++ // CPU-half and a device src is always the contiguous scheduler-dup). ++ const bool use_2d = two_region ++ ? ((size_t) tK->nb[1] != k_row || (size_t) tV->nb[1] != v_row) ++ : tr.host; ++ const int slot = t % n_slots; ++ const size_t k_nb2 = (size_t)this_kv * k_row; ++ const size_t v_nb2 = (size_t)this_kv * v_row; ++ char * sK = slotK.ptr + (size_t)slot * k_slot; ++ char * sV = slotV.ptr + (size_t)slot * v_slot; ++ if (!all_f16) { ++ // S3d dequant-on-lift: ONE nc-converter launch per tile reads the ++ // (head_dim × this_kv × n_head_kv × n_b) source sub-block at its ++ // native strides (s = nb/ts: blocks-per-row for quant, elements ++ // for f16) and writes the PACKED f16 slot. The converter's packed ++ // [ne00,ne01,ne02,ne03] output lands head/batch blocks at exactly ++ // dstoff*k_nb2 (dstoff = bb*n_head_kv+hh), so the FA/lse readers ++ // below are unchanged. Unifies device + pinned-host (S3c) sources ++ // (the kernel reads host via UVA). Slot is f16 → lse half-cast OK. ++ // 3c-5: tK/tV + loc select the window (device) or tail (host) source. ++ to_fp16_k((const char *)tK->data + (size_t)loc*tK->nb[1], (half *)sK, ++ head_dim, this_kv, n_head_kv, n_b, ++ (int64_t)(tK->nb[1]/ts_k), (int64_t)(tK->nb[2]/ts_k), (int64_t)(tK->nb[3]/ts_k), cs); ++ to_fp16_v((const char *)tV->data + (size_t)loc*tV->nb[1], (half *)sV, ++ dv, this_kv, n_head_kv, n_b, ++ (int64_t)(tV->nb[1]/ts_v), (int64_t)(tV->nb[2]/ts_v), (int64_t)(tV->nb[3]/ts_v), cs); ++ } else ++ for (int bb = 0; bb < n_b; ++bb) { ++ for (int hh = 0; hh < n_head_kv; ++hh) { ++ const size_t dstoff = (size_t)bb*n_head_kv + hh; ++ const char * srcK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)hh*tK->nb[2] + (size_t)loc*tK->nb[1]; ++ const char * srcV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)hh*tV->nb[2] + (size_t)loc*tV->nb[1]; ++ if (use_2d) { ++ // S3c: strided source (pinned-host CPU-half, OR the 3c-5 ++ // permuted device window — all heads per cell). Key rows ++ // strided by nb[1]; the 2D copy packs them into the ++ // contiguous slot (dst pitch == width == row), byte- ++ // equivalent to the device 1-D path. cudaMemcpyDefault ++ // (UVA) reads host or device transparently. ++ CUDA_CHECK(cudaMemcpy2DAsync( ++ sK + dstoff*k_nb2, k_row, srcK, tK->nb[1], k_row, this_kv, cudaMemcpyDefault, cs)); ++ CUDA_CHECK(cudaMemcpy2DAsync( ++ sV + dstoff*v_nb2, v_row, srcV, tV->nb[1], v_row, this_kv, cudaMemcpyDefault, cs)); ++ } else { ++ CUDA_CHECK(cudaMemcpyAsync(sK + dstoff*k_nb2, srcK, k_nb2, cudaMemcpyDefault, cs)); ++ CUDA_CHECK(cudaMemcpyAsync(sV + dstoff*v_nb2, srcV, v_nb2, cudaMemcpyDefault, cs)); ++ } ++ } ++ } ++ }; ++ ++ // Prologue: stage tile 0 so its compute can start as soon as it lands. ++ lift(0); ++ CUDA_CHECK(cudaEventRecord(copy_done[0], cs)); ++ ++ for (int t = 0; t < n_tiles; ++t) { ++ const int slot = t % n_slots; ++ ++ // Prefetch tile t+1 on copy_stream (overlaps tile t's compute). If ++ // its slot is being reused, first wait for the prior occupant's ++ // compute to release it (compute_done was recorded n_slots tiles ago). ++ if (t + 1 < n_tiles) { ++ const int nslot = (t + 1) % n_slots; ++ if (t + 1 >= n_slots) { ++ CUDA_CHECK(cudaStreamWaitEvent(cs, compute_done[nslot], 0)); ++ } ++ lift(t + 1); ++ CUDA_CHECK(cudaEventRecord(copy_done[nslot], cs)); ++ } ++ ++ // Compute tile t on ctx.stream(): wait for its copy, then FA + lse. ++ // 3c-5: kv0 is the GLOBAL key position (window tile: t·tile_kv_full; ++ // tail tile: n_win + local) → the mask slice below lands on the right ++ // logical column. The slot (sK/sV) was already packed by lift(t) from ++ // the region source, so the FA reads it region-agnostically. ++ const tile_region_t trc = resolve(t); ++ const int kv0 = trc.global_kv0; ++ const int this_kv = trc.this_kv; ++ float * lse_t = lse_tiles.ptr + (size_t)t * n_rows; ++ char * sK = slotK.ptr + (size_t)slot * k_slot; ++ char * sV = slotV.ptr + (size_t)slot * v_slot; ++ CUDA_CHECK(cudaStreamWaitEvent(stream, copy_done[slot], 0)); ++ ++ if (this_kv > 0) { ++ const size_t k_nb2 = (size_t)this_kv * k_row; ++ const size_t v_nb2 = (size_t)this_kv * v_row; ++ // S3d: the slot is always f16 and packed at k_row/v_row stride. ++ // Tag type=F16 (per-tile FA → (F16,F16) case) and set nb[0..1] ++ // to the packed-f16 layout. nb[1]=k_row also fixes the ++ // n_head_cpu>1 stride bug (the slot stride is k_row, not K->nb[1]); ++ // value-preserving on the f16/device path (there K->nb[1]==k_row). ++ // VALIDATED on Gemma-4 A4B (8 KV heads → frac 0.5 = 4 GPU/4 CPU, ++ // n_head_cpu=4, 25/30 layers streaming): RULER vt@4k q8_0 = 100%, ++ // byte-equal in score to resident-q8_0 ground truth (2026-06-01). ++ ggml_tensor Kt = *K; ++ Kt.type = GGML_TYPE_F16; ++ Kt.ne[1] = this_kv; Kt.data = sK; ++ Kt.nb[0] = sizeof(half); Kt.nb[1] = k_row; ++ Kt.nb[2] = k_nb2; Kt.nb[3] = k_nb2 * n_head_kv; ++ ggml_tensor Vt = *V; ++ Vt.type = GGML_TYPE_F16; ++ Vt.ne[1] = this_kv; Vt.data = sV; ++ Vt.nb[0] = sizeof(half); Vt.nb[1] = v_row; ++ Vt.nb[2] = v_nb2; Vt.nb[3] = v_nb2 * n_head_kv; ++ ggml_tensor Mt; ++ ggml_tensor * Mtp = nullptr; ++ if (mask) { ++ Mt = *mask; Mt.ne[0] = this_kv; ++ Mt.data = (char *)mask->data + (size_t)kv0*mask->nb[0]; ++ Mtp = &Mt; ++ } ++ ggml_tensor dst_t = *dst; ++ dst_t.op = GGML_OP_FLASH_ATTN_EXT; ++ dst_t.src[1] = &Kt; ++ dst_t.src[2] = &Vt; ++ dst_t.src[3] = Mtp; ++ dst_t.src[4] = nullptr; ++ dst_t.data = O_tiles.ptr + (size_t)t * nel; ++ // S3b HYBRID lse sourcing (opencoti-hook: f5-rolling-kv #340). ++ // DECODE on a finalize-guaranteed tile (n_q==1 && head_dim>256 ⇒ ++ // MMA; stream_k uniform fixup writes dst_lse for EVERY head-row): ++ // arm the consume-once channel so the per-tile FA emits lse_t ++ // itself, killing the recompute launch (the M7 launch-slack win). ++ // Else (prefill, head<=256/Qwen → VEC has no finalize) recompute ++ // with the softcap-aware streaming_lse_kernel. ++ // ++ // bug-263 (M7 Stage-3c spike, Gemma) — UNRESOLVED, see .wolf/buglog.json. ++ // WARNING: the `> 2*FATTN_KQ_STRIDE` boundary below is NOT a fix. ++ // A 9-config sweep proved decode (n_q==1, head_dim 512, GQA) with ++ // tile_kv_full == EXACTLY 512 silently corrupts output (needle lost, ++ // NO assert); 256/768/1024 all pass. The defect is INDEPENDENT of lse ++ // source — the original `>=` armed finalize at 512 (broke), and this ++ // `>` instead routes the 512 tile to streaming_lse_kernel recompute ++ // (ALSO breaks). Neither path is proven correct at this_kv∈{256,512} ++ // for multi-head GQA decode. The earlier "ntiles_KV==2 / finalize lse ++ // wrong" rationale was WRONG (Ampere head 512 uses nbatch_fa=32 → ++ // ntiles_KV=16 at 512, not 2). `>` is kept only because it is no worse ++ // than `>=`; the real mechanism is unisolated. Real Stage-3c tail tiles ++ // can be ≤512 at decode → MUST resolve before relying on this path. ++ const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ if (decode_lse) opencoti_fattn_dst_lse = lse_t; ++ ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ ++ if (!decode_lse) { ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, sK, ++ mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, ++ lse_t, scale, logit_softcap, this_kv, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], k_row, k_nb2, k_nb2 * n_head_kv, ++ mask ? mask->nb[0] : 0, mask ? mask->nb[1] : 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ } else { ++ // Empty tile (this_kv==0): the kernel writes lse=-inf and never ++ // dereferences the slot, so the K strides are unused → pass 0. ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, sK, nullptr, ++ lse_t, scale, logit_softcap, 0, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], k_row, 0, 0, 0, 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ CUDA_CHECK(cudaEventRecord(compute_done[slot], stream)); ++ } ++ } else ++ // S3a: single compute stream. The per-tile slot lift, the FA over the ++ // slot, and the lse all run on ctx.stream() — stream-ordered so the FA ++ // reads the freshly-copied slot, fully capturable under cuda_graph, and ++ // free of cross-stream collisions with M2's concurrent-event streams ++ // (which forced the earlier garbage). The overlapping double-buffer ++ // (dedicated copy_stream + events) is the S3b branch above. ++ for (int t = 0; t < n_tiles; ++t) { ++ const int slot = t % n_slots; ++ const int kv0 = t * tile_kv_full; ++ const int this_kv = kv0 >= n_kv ? 0 : min(tile_kv_full, n_kv - kv0); ++ float * lse_t = lse_tiles.ptr + (size_t)t * n_rows; ++ char * sK = slotK.ptr + (size_t)slot * k_slot; ++ char * sV = slotV.ptr + (size_t)slot * v_slot; ++ ++ if (this_kv > 0) { ++ const size_t k_nb2 = (size_t)this_kv * k_row; ++ const size_t v_nb2 = (size_t)this_kv * v_row; ++ // Lift the tile into the packed slot. For a DEVICE source the ++ // scheduler's device-dup makes each (b,h)'s this_kv key rows ++ // contiguous (nb[1]==k_row) → one packed 1-D copy each (sidesteps ++ // cudaMemcpy2D pitch limits; nb[2] of the source is < this_kv*k_row ++ // when ne[2]==1). For a HOST source (S3c bypass — the permuted ++ // pinned-host view, nb[1]==row(n_embd_cpu) != k_row), the rows are ++ // strided by the full per-token n_embd; a packed 1-D copy would ++ // scoop adjacent heads → 2-D lift (spitch=nb[1], width=k_row). ++ // cudaMemcpyDefault: UVA infers H2D (pinned spill) vs D2D. ++ // opencoti-hook: f5-rolling-kv — S3c scheduler-bypass host lift; ++ // S3d dequant-on-lift (the !all_f16 branch). ++ if (!all_f16) { ++ // S3d: one nc-converter launch per tile → packed f16 slot (see ++ // the overlap path for the layout proof). Source strides s=nb/ts. ++ to_fp16_k((const char *)K->data + (size_t)kv0*K->nb[1], (half *)sK, ++ head_dim, this_kv, n_head_kv, n_b, ++ (int64_t)(K->nb[1]/ts_k), (int64_t)(K->nb[2]/ts_k), (int64_t)(K->nb[3]/ts_k), stream); ++ to_fp16_v((const char *)V->data + (size_t)kv0*V->nb[1], (half *)sV, ++ dv, this_kv, n_head_kv, n_b, ++ (int64_t)(V->nb[1]/ts_v), (int64_t)(V->nb[2]/ts_v), (int64_t)(V->nb[3]/ts_v), stream); ++ } else ++ for (int bb = 0; bb < n_b; ++bb) { ++ for (int hh = 0; hh < n_head_kv; ++hh) { ++ const size_t dstoff = (size_t)bb*n_head_kv + hh; ++ const char * srcK = (const char *)K->data + (size_t)bb*K->nb[3] + (size_t)hh*K->nb[2] + (size_t)kv0*K->nb[1]; ++ const char * srcV = (const char *)V->data + (size_t)bb*V->nb[3] + (size_t)hh*V->nb[2] + (size_t)kv0*V->nb[1]; ++ if (host_src) { ++ CUDA_CHECK(cudaMemcpy2DAsync( ++ sK + dstoff*k_nb2, k_row, srcK, K->nb[1], k_row, this_kv, cudaMemcpyDefault, stream)); ++ CUDA_CHECK(cudaMemcpy2DAsync( ++ sV + dstoff*v_nb2, v_row, srcV, V->nb[1], v_row, this_kv, cudaMemcpyDefault, stream)); ++ } else { ++ CUDA_CHECK(cudaMemcpyAsync(sK + dstoff*k_nb2, srcK, k_nb2, cudaMemcpyDefault, stream)); ++ CUDA_CHECK(cudaMemcpyAsync(sV + dstoff*v_nb2, srcV, v_nb2, cudaMemcpyDefault, stream)); ++ } ++ } ++ } ++ ++ // FA over the packed slot. S3d: slot is always f16 at k_row/v_row ++ // stride → tag type=F16, nb[0..1] to the packed-f16 layout (nb[1]= ++ // k_row also fixes the n_head_cpu>1 stride bug — VALIDATED on ++ // Gemma-4 n_head_cpu=4, see overlap path above; value-preserving ++ // on f16/device where K->nb[1]==k_row). ++ ggml_tensor Kt = *K; ++ Kt.type = GGML_TYPE_F16; ++ Kt.ne[1] = this_kv; Kt.data = sK; ++ Kt.nb[0] = sizeof(half); Kt.nb[1] = k_row; ++ Kt.nb[2] = k_nb2; Kt.nb[3] = k_nb2 * n_head_kv; ++ ggml_tensor Vt = *V; ++ Vt.type = GGML_TYPE_F16; ++ Vt.ne[1] = this_kv; Vt.data = sV; ++ Vt.nb[0] = sizeof(half); Vt.nb[1] = v_row; ++ Vt.nb[2] = v_nb2; Vt.nb[3] = v_nb2 * n_head_kv; ++ ggml_tensor Mt; ++ ggml_tensor * Mtp = nullptr; ++ if (mask) { ++ Mt = *mask; Mt.ne[0] = this_kv; ++ Mt.data = (char *)mask->data + (size_t)kv0*mask->nb[0]; ++ Mtp = &Mt; ++ } ++ ggml_tensor dst_t = *dst; ++ dst_t.op = GGML_OP_FLASH_ATTN_EXT; ++ dst_t.src[1] = &Kt; ++ dst_t.src[2] = &Vt; ++ dst_t.src[3] = Mtp; ++ dst_t.src[4] = nullptr; ++ dst_t.data = O_tiles.ptr + (size_t)t * nel; ++ // S3a HYBRID lse sourcing (opencoti-hook: f5-rolling-kv #340) — see ++ // the S3b overlap path for the full bug-263 note. DECODE on a ++ // finalize tile arms the channel so the FA emits lse_t; else recompute ++ // (softcap-aware). The `> 2*FATTN_KQ_STRIDE` boundary is NOT a fix for ++ // bug-263 (UNRESOLVED: decode tile_kv_full==512 corrupts on either lse ++ // path) — kept only as no-worse-than-`>=`. See .wolf/buglog.json. ++ const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ if (decode_lse) opencoti_fattn_dst_lse = lse_t; ++ ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ ++ if (!decode_lse) { ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, sK, ++ mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, ++ lse_t, scale, logit_softcap, this_kv, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], k_row, k_nb2, k_nb2 * n_head_kv, ++ mask ? mask->nb[0] : 0, mask ? mask->nb[1] : 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ } else { ++ // Empty trailing tile: lse=-inf, combine skips it (no copy/FA). The ++ // kernel never dereferences the slot, so K strides are unused → 0. ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, sK, nullptr, ++ lse_t, scale, logit_softcap, 0, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], k_row, 0, 0, 0, 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ } ++ } else { ++ // S2 resident-view path (default; byte-identical gate). ++ for (int t = 0; t < n_tiles; ++t) { ++ const int kv0 = t * tile_kv_full; ++ float * lse_t = lse_tiles.ptr + (size_t)t * n_rows; ++ const int this_kv = kv0 >= n_kv ? 0 : min(tile_kv_full, n_kv - kv0); ++ ++ // S2 HYBRID lse sourcing (opencoti-hook: f5-rolling-kv #340): DECODE on a ++ // finalize tile arms the channel so the FA emits lse_t; the recompute below ++ // is then skipped. Prefill / head<=256 (VEC, no finalize) / empty tiles fall ++ // to the recompute (which also writes the -inf fill). The `> 2*FATTN_KQ_STRIDE` ++ // boundary is NOT a fix for bug-263 (UNRESOLVED: decode tile_kv_full==512 ++ // corrupts on either lse path) — kept only as no-worse-than-`>=`. See the ++ // S3b overlap path's full note + .wolf/buglog.json. ++ const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ if (this_kv > 0) { ++ // FA over the tile's key-slice → O_t (stock kernel, untouched). ++ ggml_tensor Kt = *K; Kt.ne[1] = this_kv; Kt.data = (char *)K->data + (size_t)kv0*K->nb[1]; ++ ggml_tensor Vt = *V; Vt.ne[1] = this_kv; Vt.data = (char *)V->data + (size_t)kv0*V->nb[1]; ++ ggml_tensor Mt; ++ ggml_tensor * Mtp = nullptr; ++ if (mask) { ++ Mt = *mask; Mt.ne[0] = this_kv; Mt.data = (char *)mask->data + (size_t)kv0*mask->nb[0]; ++ Mtp = &Mt; ++ } ++ ggml_tensor dst_t = *dst; ++ dst_t.op = GGML_OP_FLASH_ATTN_EXT; // bona-fide FA op for the sub-call ++ dst_t.src[1] = &Kt; ++ dst_t.src[2] = &Vt; ++ dst_t.src[3] = Mtp; ++ dst_t.src[4] = nullptr; ++ dst_t.data = O_tiles.ptr + (size_t)t * nel; ++ if (decode_lse) opencoti_fattn_dst_lse = lse_t; ++ ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ } ++ ++ // lse_t (this_kv==0 → empty tile: kernel writes -inf, combine ignores it). ++ if (!decode_lse) { ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, ++ (const char *)K->data + (size_t)kv0*K->nb[1], ++ mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, ++ lse_t, scale, logit_softcap, this_kv, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], ++ mask ? mask->nb[0] : 0, mask ? mask->nb[1] : 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ } ++ } ++ ++ const int64_t total = (int64_t)dv * n_rows; ++ const int threads = 256; ++ const int64_t blocks = (total + threads - 1) / threads; ++ ++ // S3-b (#353) CPU_FA_TAIL: after the window tiles [0,n_tiles), convert the ++ // precomputed host partial into the final combine tile [n_tiles]. The partial ++ // (src[5] = [2+DV, n_rows] [M,S,VKQ]) is staged to device with a single UVA ++ // cudaMemcpyAsync — only this tiny buffer (≪ the tail K/V) crosses PCIe. The ++ // window leg already filled tiles [0,n_tiles) above; the combine below then ++ // merges window ⊕ tail through the unchanged online-softmax reduction. ++ if (tail_is_partial) { ++ const size_t pcount = (size_t)(2 + dv) * (size_t)n_rows; ++ ggml_cuda_pool_alloc p_dev(pool, pcount); ++ CUDA_CHECK(cudaMemcpyAsync(p_dev.ptr, tail_partial->data, pcount * sizeof(float), ++ cudaMemcpyDefault, stream)); ++ tail_partial_to_tile_kernel<<<(unsigned)blocks, threads, 0, stream>>>( ++ p_dev.ptr, O_tiles.ptr + (size_t)n_tiles * nel, ++ lse_tiles.ptr + (size_t)n_tiles * n_rows, dv, n_rows); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ ++ streaming_combine_kernel<<<(unsigned)blocks, threads, 0, stream>>>( ++ O_tiles.ptr, lse_tiles.ptr, (float *)dst->data, n_combine_tiles, dv, n_rows); ++ CUDA_CHECK(cudaGetLastError()); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +@@ -3,3 +3,9 @@ + void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + + bool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst); ++ ++// opencoti-hook: f5-rolling-kv — W4 M7-D #304 / R2-b S2 (#311). ++// Streaming flash-attention: key-axis tiling + LSE-weighted online-softmax ++// combine over the stock FA kernels (which stay untouched). Falls back to ++// plain FA for the single-tile / unsupported-scoring cases. ++void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -103,6 +103,11 @@ void ggml_cuda_error(const char * stmt, const char * func, const char * file, in + GGML_LOG_ERROR(GGML_CUDA_NAME " error: %s\n", msg); + GGML_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func, file, line); + GGML_LOG_ERROR(" %s\n", stmt); ++ // opencoti: the llamafile log callback swallows GGML_LOG_ERROR on the abort ++ // path, so the failing stmt/msg never reaches the server log. Mirror it to ++ // stderr directly (flushed) so CUDA faults are diagnosable. Harmless. ++ fprintf(stderr, "[opencoti CUDA] error: %s | stmt: %s | %s:%d (dev %d)\n", msg, stmt, file, line, id); ++ fflush(stderr); + // abort with GGML_ABORT to get a stack trace + GGML_ABORT(GGML_CUDA_NAME " error"); + } +@@ -3024,6 +3029,16 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg + ggml_cuda_mul_mat_vec_q(ctx, up_exps, cur, ids, dst, &fusion_data); + } + break; ++ case GGML_OP_STREAMING_FLASH_ATTN: ++ // opencoti-hook: f5-rolling-kv — W4 M7-D #304 / R2-b S2 (#311). ++ // Key-axis tiling + LSE-weighted online-softmax over the STOCK FA ++ // kernels (left untouched). Single-tile short-circuits to plain FA ++ // (= R2-b S1, byte-identical). NB: the mma kernel our 3090 selects ++ // normalises in-kernel and combines via stream_k (NOT ++ // flash_attn_combine_results), so the cross-tile combine is owned ++ // here in F32, not delegated to the kernel. See fattn.cu. ++ ggml_cuda_streaming_flash_attn(ctx, dst); ++ break; + case GGML_OP_OUT_PROD: + ggml_cuda_out_prod(ctx, dst); + break; +@@ -5579,6 +5594,35 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + #endif // GGML_USE_MUSA + case GGML_OP_FLASH_ATTN_EXT: + return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op); ++ case GGML_OP_STREAMING_FLASH_ATTN: { ++ // opencoti-hook: f5-rolling-kv — bug-259. The streaming forward repacks ++ // its (possibly non-contiguous / pinned-host) K/V into CONTIGUOUS f16 ++ // slots before each tile's stock FA, so delegating to ++ // ggml_cuda_flash_attn_ext_supported is WRONG: that check rejects any ++ // src whose nb[i] is not 16-aligned (true for the S3c host-permuted ++ // views and for some -fit/reserve trial graphs), which made the op ++ // report unsupported on CUDA at boot → it fell to the CPU backend (no ++ // kernel → abort) or to no backend (assignment assert). Validate only ++ // what the repacked slot FA actually needs — a supported head_dim with ++ // a matching V width; K/V storage type is irrelevant (dequantized to ++ // f16 on the lift). Scoring (max_bias/softcap/sinks) is handled by the ++ // op's own resident fallback at run time. ++ const ggml_tensor * K = op->src[1]; ++ const ggml_tensor * V = op->src[2]; ++ const int64_t dk = K->ne[0]; ++ const int64_t dv = V->ne[0]; ++ switch (dk) { ++ case 40: case 64: case 72: case 80: case 96: ++ case 112: case 128: case 256: ++ return dv == dk; ++ case 512: ++ return dv == dk; ++ case 576: ++ return dv == 512; ++ default: ++ return false; ++ } ++ } + case GGML_OP_CROSS_ENTROPY_LOSS: + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + case GGML_OP_OPT_STEP_ADAMW: +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -1079,9 +1079,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { + + "GLU", + "MOE_FUSED_UP_GATE", ++ "STREAMING_FLASH_ATTN", ++ "FLASH_ATTN_TAIL_PARTIAL", + }; + +-static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97"); ++static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); + + static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "none", +@@ -1190,9 +1192,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + + "glu(x)", + "moe_up_gate(up,gate,x)", ++ "streaming_flash_attn(q,k,v)", ++ "flash_attn_tail_partial(q,k,v)", + }; + +-static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97"); ++static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); + + static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); + +@@ -5419,10 +5423,166 @@ struct ggml_tensor * ggml_flash_attn_ext( + return result; + } + ++// opencoti F5 M7 Rolling KV (W4 M7-D #304) — streaming flash attention. ++// Construction mirrors ggml_flash_attn_ext exactly (same src layout, same ++// op_params {scale, max_bias, logit_softcap}, same F32 permuted output) so ++// the CUDA forward can reuse the proven FA pre/post-processing and only ++// differ in how it sweeps the key dimension (tiled + online-softmax). The ++// op carries no extra params at R2-a; tile/slot sizing is decided inside the ++// CUDA forward from device free-VRAM. Precision is forced to F32 by the ++// graph builder via ggml_flash_attn_ext_set_prec (shared setter). ++struct ggml_tensor * ggml_streaming_flash_attn( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap) { ++ GGML_ASSERT(ggml_can_mul_mat(k, q)); ++ ++ GGML_ASSERT(q->ne[3] == k->ne[3]); ++ GGML_ASSERT(q->ne[3] == v->ne[3]); ++ ++ if (mask) { ++ GGML_ASSERT(mask->type == GGML_TYPE_F16); ++ GGML_ASSERT(ggml_is_contiguous(mask)); ++ GGML_ASSERT(q->ne[2] % mask->ne[2] == 0); ++ GGML_ASSERT(q->ne[3] % mask->ne[3] == 0); ++ } ++ ++ if (max_bias > 0.0f) { ++ GGML_ASSERT(mask); ++ } ++ ++ // permute(0, 2, 1, 3) — identical to flash_attn_ext ++ int64_t ne[4] = { v->ne[0], q->ne[2], q->ne[1], q->ne[3] }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ float params[] = { scale, max_bias, logit_softcap }; ++ ggml_set_op_params(result, params, sizeof(params)); ++ ++ result->op = GGML_OP_STREAMING_FLASH_ATTN; ++ result->src[0] = q; ++ result->src[1] = k; ++ result->src[2] = v; ++ result->src[3] = mask; ++ ++ return result; ++} ++ ++// opencoti F5 M7 Stage 3c-5 (#341) — position-window streaming FA. Same op as ++// ggml_streaming_flash_attn but with the resident device window K/V in src[1]/ ++// src[2] and the pinned-host overflow tail K/V in src[5]/src[6]. The CUDA forward ++// tiles the window (device, in-place/D2D) AND the tail (host, H2D-streamed) into ++// ONE online-softmax combine, so the bulk stays resident while only the overflow ++// streams. mask spans the full logical key range [0, n_win + n_tail). tail null ⇒ ++// identical to ggml_streaming_flash_attn (single resident region). ++struct ggml_tensor * ggml_streaming_flash_attn_window( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k_win, ++ struct ggml_tensor * v_win, ++ struct ggml_tensor * k_tail, ++ struct ggml_tensor * v_tail, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap) { ++ GGML_ASSERT(ggml_can_mul_mat(k_win, q)); ++ GGML_ASSERT(q->ne[3] == k_win->ne[3]); ++ GGML_ASSERT(q->ne[3] == v_win->ne[3]); ++ if (k_tail) { ++ GGML_ASSERT(v_tail); ++ GGML_ASSERT(k_tail->ne[0] == k_win->ne[0]); ++ GGML_ASSERT(v_tail->ne[0] == v_win->ne[0]); ++ GGML_ASSERT(k_tail->ne[2] == k_win->ne[2]); ++ } ++ if (mask) { ++ GGML_ASSERT(mask->type == GGML_TYPE_F16); ++ GGML_ASSERT(ggml_is_contiguous(mask)); ++ GGML_ASSERT(q->ne[2] % mask->ne[2] == 0); ++ GGML_ASSERT(q->ne[3] % mask->ne[3] == 0); ++ } ++ if (max_bias > 0.0f) { ++ GGML_ASSERT(mask); ++ } ++ ++ int64_t ne[4] = { v_win->ne[0], q->ne[2], q->ne[1], q->ne[3] }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ float params[] = { scale, max_bias, logit_softcap }; ++ ggml_set_op_params(result, params, sizeof(params)); ++ ++ result->op = GGML_OP_STREAMING_FLASH_ATTN; ++ result->src[0] = q; ++ result->src[1] = k_win; ++ result->src[2] = v_win; ++ result->src[3] = mask; ++ // src[4] reserved for sinks (ggml_flash_attn_ext_add_sinks) ++ result->src[5] = k_tail; ++ result->src[6] = v_tail; ++ ++ return result; ++} ++ ++// opencoti F5 M7 Stage 3d-S3 (#353) — CPU tail-partial emitter for the CPU_FA_TAIL ++// tactic. Runs flash-attention over a (host-resident) tail K/V in ONE online-softmax ++// pass and writes the UN-normalized partial [M, S, VKQ] per output row; dst shape is ++// [2+DV, n_head, n_q, n_b]. That is exactly the partial the CUDA online-softmax combine ++// (fattn.cu streaming_combine_kernel) consumes after a trivial per-row normalize: ++// O_row = VKQ_row / S_row , lse_row = M_row + logf(S_row). ++// The window leg's (O,lse) is computed on-device by the streaming op; only this tiny ++// partial (2+DV floats per head) crosses PCIe instead of the whole tail K/V — the ++// CPU_FA_TAIL memory win on slow links. NO sinks here (src[4] stays null): sinks are ++// single-counted on the window partial only. Decode-only consumer (n_q==1): the CPU ++// per-row order (q-fastest within head) coincides with the combine's head-fastest row ++// only when n_q==1; the dispatch routes CPU_FA_TAIL for decode and falls back otherwise. ++struct ggml_tensor * ggml_flash_attn_ext_tail_partial( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap) { ++ GGML_ASSERT(ggml_can_mul_mat(k, q)); ++ GGML_ASSERT(q->ne[3] == k->ne[3]); ++ GGML_ASSERT(q->ne[3] == v->ne[3]); ++ if (mask) { ++ GGML_ASSERT(mask->type == GGML_TYPE_F16); ++ GGML_ASSERT(ggml_is_contiguous(mask)); ++ GGML_ASSERT(q->ne[2] % mask->ne[2] == 0); ++ GGML_ASSERT(q->ne[3] % mask->ne[3] == 0); ++ } ++ if (max_bias > 0.0f) { ++ GGML_ASSERT(mask); ++ } ++ ++ // dst rows = (head, q-token, batch); the 2 leading slots hold (M, S) ahead of VKQ. ++ int64_t ne[4] = { 2 + v->ne[0], q->ne[2], q->ne[1], q->ne[3] }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ float params[] = { scale, max_bias, logit_softcap }; ++ ggml_set_op_params(result, params, sizeof(params)); ++ ++ result->op = GGML_OP_FLASH_ATTN_TAIL_PARTIAL; ++ result->src[0] = q; ++ result->src[1] = k; ++ result->src[2] = v; ++ result->src[3] = mask; ++ ++ return result; ++} ++ + void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec) { +- GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); ++ // opencoti F5 M7 Rolling KV (W4 M7-D #304) — streaming-FA shares FA's ++ // op_params layout (prec at index 3), so this setter serves both. ++ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT || a->op == GGML_OP_STREAMING_FLASH_ATTN); + + const int32_t prec_i32 = (int32_t) prec; + +@@ -5431,7 +5591,7 @@ void ggml_flash_attn_ext_set_prec( + + enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a) { +- GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); ++ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT || a->op == GGML_OP_STREAMING_FLASH_ATTN); + + const int32_t prec_i32 = ggml_get_op_params_i32(a, 3); + +@@ -5446,7 +5606,7 @@ void ggml_flash_attn_ext_add_sinks( + return; + } + +- GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT); ++ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT || a->op == GGML_OP_STREAMING_FLASH_ATTN); + GGML_ASSERT(a->src[4] == NULL); + GGML_ASSERT(a->src[0]->ne[2] == sinks->ne[0]); + GGML_ASSERT(sinks->type == GGML_TYPE_F32); +diff --git a/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp b/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp +--- a/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp ++++ b/llama.cpp/ggml/src/iqk/iqk_flash_attn.cpp +@@ -138,6 +138,60 @@ static inline bool are_kv_types_supported(ggml_type type_k, ggml_type type_v) { + return it_k != supported.end() && it_v != supported.end(); + } + ++// opencoti-hook: f5-rolling-kv — W4 M7 S3 (#354). iqk-backed CPU tail flash-attention ++// partial for the CPU_FA_TAIL rolling-KV tactic. See ggml-include/ggml-iqk-flash-attn.h. ++// One q row (nq1=1) over the host tail; emits the UN-normalized [M, S, VKQ] partial the ++// CUDA streaming combine consumes. CONVENTION (verified fa/iqk_fa_templates.h:1185 ++// normalize_and_store): when M AND S are non-null, iqk_flash_attn_impl returns the ++// UN-normalized qkv directly (memcpy of qkv_cache = Sum exp(s-M)*V, NO divide by S) plus ++// (M=max logit, S=sum exp(s-M)) — exactly the [M,S,VKQ] the combine wants (O=VKQ/S, ++// lse=M+logS), byte-compatible with ops.cpp _f16_one_chunk write_partials. So we copy qkv ++// straight through; do NOT multiply by S (that double-counts and produces garbage — the ++// bug-354 niah=0.0). softcap is folded by the ++// caller (scale/=softcap when softcap!=0) exactly as the iqk_flash_attn_noalibi call site; ++// for Gemma-4 global layers softcap==0. No sinks here (single-counted on the GPU window ++// leg). Returns false on unsupported (type, shape) so the caller falls back to ++// _f16_one_chunk. The ggml thread partition (one row per call) replaces iqk's internal ++// threading: iqk_flash_attn_impl is the single-call kernel (no barrier / wdata). ++extern "C" IQK_API bool iqk_flash_attn_tail_partial(int type_k, int type_v, int Dk, int Dv, ++ int nk1, int stride_k, int stride_v, int stride_m, ++ const float * q, const void * k, const void * v, const void * mask, ++ float scale, float softcap, float * partial) { ++ if (!mask || nk1 % 32 != 0) { ++ return false; // iqk assumes a non-null mask and nk multiple of 32 ++ } ++ if (Dv > 512) { ++ return false; // qkv scratch bound below ++ } ++ if (!are_kv_types_supported((ggml_type) type_k, (ggml_type) type_v)) { ++ return false; ++ } ++ ++ float qkv[512]; // UN-normalized Sum exp(s-M)*V for the single row (M,S non-null mode) ++ float M = -INFINITY; // running max logit (scaled+masked) ++ float S = 0.0f; // sum exp(logit - M) ++ ++ const bool ok = iqk_flash_attn_impl( ++ type_k, type_v, Dk, Dv, ++ /*nq1=*/1, nk1, ++ /*stride_q=*/Dk * (int) sizeof(float), stride_k, stride_v, stride_m, ++ /*stride_qkv=*/Dv, ++ q, k, v, mask, /*sinksf=*/nullptr, /*nsinks=*/0, ++ scale, softcap, qkv, &M, &S); ++ if (!ok) { ++ return false; ++ } ++ ++ partial[0] = M; ++ partial[1] = S; ++ // qkv is ALREADY the un-normalized VKQ (M,S non-null path of normalize_and_store) — copy ++ // straight through. (The previous `qkv[d] * S` double-counted S → niah=0.0, bug-354.) ++ for (int d = 0; d < Dv; ++d) { ++ partial[2 + d] = qkv[d]; ++ } ++ return true; ++} ++ + // TODO: get the ggml_type enum here without polution + // + extern "C" IQK_API bool iqk_flash_attn_noalibi(int type_q, int type_mask, float max_bias, +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -358,6 +358,17 @@ extern "C" { + // streamed back per step). 1.0 = off. Only effective for offloaded + // layers with flash-attention (non-transposed V cache). + float headinfer_gpu_heads_frac; ++ // opencoti F5 M7 Rolling KV — Rung 0 auto residency — see docs/features/rolling_kv.md ++ // VRAM budget cap (MiB) for auto KV residency. 0 = use all free VRAM minus a ++ // compute reserve (maximize). Consulted only when headinfer_gpu_heads_frac < 0. ++ uint32_t vram_target_mib; ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304). Effective H2D PCIe ++ // bandwidth (GB/s) for streaming-FA tile sizing. 0 = unknown. ++ float pcie_bw_gbps; ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. ++ // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head ++ // (force M2 split), 2 = window (force POSITION_WINDOW, warn+fallback if ineligible). ++ uint32_t kv_residency_mode; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + // 0 = off (default), 1 = on, 2 = auto. Effective only when the + // M2 head split is active for a given layer. +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -64,6 +64,12 @@ llama_context::llama_context( + cparams.slot_shrink_idle_ms = params.slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac = params.headinfer_gpu_heads_frac; ++ // opencoti F5 M7 Rolling KV — Rung 0 — see docs/features/rolling_kv.md ++ cparams.vram_target_mib = params.vram_target_mib; ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304) — see docs/features/rolling_kv.md ++ cparams.pcie_bw_gbps = params.pcie_bw_gbps; ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md ++ cparams.kv_residency_mode = params.kv_residency_mode; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + cparams.neo_pipeline_mode = params.neo_pipeline_mode; + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md +@@ -485,7 +491,21 @@ void llama_context::sched_reserve() { + ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); + + // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer +- GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0); ++ // opencoti-hook: f5-rolling-kv — see docs/features/rolling_kv.md. ++ // The M2 head-split (0020), M3 NEO (0030), and M7 GPU_STREAM (0070) ++ // paths emit per-layer split FA ops named "__fattn___g-" / ++ // "__fattn___c-" — two FA ops per split layer, intentionally on ++ // DIFFERENT devices (the CPU half runs the iqk CPU-FA engine). ++ // Upstream's auto-FA resolver assumes exactly one "__fattn__-" ++ // op per layer on that layer's device, so a split op both fails this ++ // name parse (the original GGML_ASSERT crashed at boot under ++ // --flash-attn auto) and would trip a false device-mismatch that ++ // disables FA. Skip any FA op that is not the plain "__fattn__-" ++ // form; the split ops are validated by their own construction gates, ++ // which require FA. Non-split layers still get the device check. ++ if (strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) != 0) { ++ continue; ++ } + const int il = std::stoi(n->name + prefix_len); + ggml_backend_dev_t device_kv = model.dev_layer(il); + if (device_fa != device_kv) { +@@ -3360,6 +3380,9 @@ llama_context_params llama_context_default_params() { + /*.slot_initial_ctx =*/ 0, + /*.slot_shrink_idle_ms =*/ 0, + /*.headinfer_gpu_heads_frac =*/ 1.0f, ++ /*.vram_target_mib =*/ 0, // opencoti F5 M7 Rolling KV — 0 = maximize ++ /*.pcie_bw_gbps =*/ 0.0f, // opencoti F5 M7 Rolling KV — 0 = unknown ++ /*.kv_residency_mode =*/ 0, // opencoti F5 M7 Rolling KV — 0 = auto + /*.neo_pipeline_mode =*/ 0, // opencoti F5 M3 neo — off by default + /*.fused_moe_up_gate =*/ false, // opencoti F5 W3 #292 — off by default + /*.n_rs_seq =*/ 0, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -23,6 +23,18 @@ struct llama_cparams { + uint32_t slot_shrink_idle_ms; + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + float headinfer_gpu_heads_frac; ++ // opencoti F5 M7 Rolling KV — Rung 0 auto residency — see docs/features/rolling_kv.md ++ // VRAM budget cap (MiB) for auto KV residency. 0 = use all free VRAM minus a ++ // compute reserve (maximize). Consulted only when headinfer_gpu_heads_frac < 0. ++ uint32_t vram_target_mib; ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304). Effective H2D PCIe ++ // bandwidth (GB/s), resolved at boot from the ReBAR probe / sysfs link ++ // (common/pcie-profile). Read at the streaming-FA dispatch to size tiles ++ // and the double-buffer; 0 = unknown (kernel uses a conservative default). ++ float pcie_bw_gbps; ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. ++ // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head, 2 = window. ++ uint32_t kv_residency_mode; + // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention. + // 0 = off (M2 concat path runs; binary equivalent to upstream when + // headinfer_gpu_heads_frac == 1.0). +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2113,6 +2113,210 @@ ggml_tensor * llm_graph_context::build_attn_mha( + return cur; + } + ++// opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304), S1. GPU_STREAM tactic. ++// Structurally identical to build_attn_mha_neo: the device head-group runs a ++// resident ggml_flash_attn_ext, the host (spilled) head-group runs the custom ++// GGML_OP_STREAMING_FLASH_ATTN op, and the two outputs are concatenated on the ++// head axis. At S1 the streaming op's CUDA forward is still ++// ggml_cuda_flash_attn_ext verbatim, so the output is BYTE-IDENTICAL to the NEO ++// path (which builds the same graph with a plain flash_attn_ext on the host ++// half; NEO's stream-pair registration only affects scheduling, not output) — ++// that is the S1 gate. S2 makes the streaming op tile the host head-group's KEY ++// dimension into a device slot pool with a cross-tile online-softmax (running ++// max + denom in F32), so the spilled KV need not fit one resident slot; S3 ++// adds the copy/compute double-buffer. The host head-group is already ++// pinned-host (the M2 split allocates k_cpu/v_cpu with the pinned cpu_buft), so ++// streaming rides the M2 residency — no whole-layer host reorder. ++ggml_tensor * llm_graph_context::build_attn_mha_streaming( ++ ggml_tensor * q, ++ ggml_tensor * k_g, ++ ggml_tensor * v_g, ++ ggml_tensor * k_c, ++ ggml_tensor * v_c, ++ uint32_t gpu_heads, ++ ggml_tensor * kq_b, ++ ggml_tensor * kq_mask, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il) const { ++ GGML_ASSERT(cparams.flash_attn && "GPU_STREAM tactic requires flash-attention"); ++ GGML_ASSERT(kq_b == nullptr && "streaming flash attention does not support KQ bias"); ++ GGML_ASSERT(v_mla == nullptr && "streaming flash attention does not support MLA"); ++ GGML_ASSERT(gpu_heads > 0 && "GPU_STREAM requires an active head split"); ++ GGML_ASSERT(k_g && v_g && k_c && v_c); ++ ++ GGML_ASSERT(k_g->ne[3] == k_c->ne[3]); ++ GGML_ASSERT(v_g->ne[3] == v_c->ne[3]); ++ GGML_ASSERT(k_g->nb[1] <= k_g->nb[2] && "headinfer split requires non-transposed V"); ++ ++ // Split Q on the head axis at the gpu/cpu boundary (mirrors ++ // build_attn_mha_neo). GQA: Q-heads serving one KV head are contiguous in ++ // dim 1, so each half is a clean view. ++ const uint32_t n_head_q = (uint32_t) q->ne[1]; ++ const uint32_t n_head_kv = (uint32_t) (k_g->ne[1] + k_c->ne[1]); ++ GGML_ASSERT(n_head_q % n_head_kv == 0 && "GQA ratio must divide n_head_q"); ++ const uint32_t gqa_ratio = n_head_q / n_head_kv; ++ const uint32_t gpu_q_heads = gpu_heads * gqa_ratio; ++ const uint32_t cpu_q_heads = (n_head_kv - gpu_heads) * gqa_ratio; ++ GGML_ASSERT(gpu_q_heads + cpu_q_heads == n_head_q); ++ ++ const auto n_stream = k_g->ne[3]; ++ ggml_tensor * q_4 = ggml_view_4d(ctx0, q, ++ q->ne[0], q->ne[1], q->ne[2]/n_stream, n_stream, ++ q->nb[1], q->nb[2], q->nb[3]/n_stream, 0); ++ ++ ggml_tensor * q_g = ggml_view_4d(ctx0, q_4, ++ q_4->ne[0], gpu_q_heads, q_4->ne[2], q_4->ne[3], ++ q_4->nb[1], q_4->nb[2], q_4->nb[3], 0); ++ ggml_tensor * q_c = ggml_view_4d(ctx0, q_4, ++ q_4->ne[0], cpu_q_heads, q_4->ne[2], q_4->ne[3], ++ q_4->nb[1], q_4->nb[2], q_4->nb[3], ++ (size_t) gpu_q_heads * q_4->nb[1]); ++ ++ q_g = ggml_permute(ctx0, q_g, 0, 2, 1, 3); ++ q_c = ggml_permute(ctx0, q_c, 0, 2, 1, 3); ++ ggml_tensor * k_g_p = ggml_permute(ctx0, k_g, 0, 2, 1, 3); ++ ggml_tensor * v_g_p = ggml_permute(ctx0, v_g, 0, 2, 1, 3); ++ ggml_tensor * k_c_p = ggml_permute(ctx0, k_c, 0, 2, 1, 3); ++ ggml_tensor * v_c_p = ggml_permute(ctx0, v_c, 0, 2, 1, 3); ++ ++ if (k_g_p->type == GGML_TYPE_F32) k_g_p = ggml_cast(ctx0, k_g_p, GGML_TYPE_F16); ++ if (k_c_p->type == GGML_TYPE_F32) k_c_p = ggml_cast(ctx0, k_c_p, GGML_TYPE_F16); ++ if (v_g_p->type == GGML_TYPE_F32) v_g_p = ggml_cast(ctx0, v_g_p, GGML_TYPE_F16); ++ if (v_c_p->type == GGML_TYPE_F32) v_c_p = ggml_cast(ctx0, v_c_p, GGML_TYPE_F16); ++ ++ ggml_tensor * sinks_g = nullptr; ++ ggml_tensor * sinks_c = nullptr; ++ if (sinks) { ++ GGML_ASSERT((uint32_t) sinks->ne[0] == n_head_q); ++ sinks_g = ggml_view_1d(ctx0, sinks, gpu_q_heads, 0); ++ sinks_c = ggml_view_1d(ctx0, sinks, cpu_q_heads, ++ (size_t) gpu_q_heads * sinks->nb[0]); ++ } ++ ++ const float alibi_bias = hparams.f_max_alibi_bias; ++ const float logit_cap = hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f; ++ ++ // Device head-group: ordinary resident flash-attention (no DMA). ++ ggml_tensor * cur_g = ggml_flash_attn_ext(ctx0, q_g, k_g_p, v_g_p, kq_mask, kq_scale, alibi_bias, logit_cap); ++ cb(cur_g, LLAMA_TENSOR_NAME_FATTN "_g", il); ++ ggml_flash_attn_ext_add_sinks(cur_g, sinks_g); ++ ggml_flash_attn_ext_set_prec (cur_g, GGML_PREC_F32); ++ ++ // Host (spilled) head-group: the streaming-FA op. At S1 its CUDA forward is ++ // plain FA, so cur_c == NEO's cur_c byte-for-byte; S2 tiles the key dim. ++ ggml_tensor * cur_c = ggml_streaming_flash_attn(ctx0, q_c, k_c_p, v_c_p, kq_mask, kq_scale, alibi_bias, logit_cap); ++ cb(cur_c, LLAMA_TENSOR_NAME_FATTN "_c", il); ++ ggml_flash_attn_ext_add_sinks(cur_c, sinks_c); ++ ggml_flash_attn_ext_set_prec (cur_c, GGML_PREC_F32); ++ ++ // Concatenate on the q-head axis (dim 1), same as build_attn_mha_neo. ++ ggml_tensor * cur = ggml_concat(ctx0, cur_g, cur_c, 1); ++ cb(cur, LLAMA_TENSOR_NAME_FATTN, il); ++ ++ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); ++ ++ return cur; ++} ++ ++// opencoti F5 M7 Stage 3c-5 (#348) — POSITION_WINDOW tactic. No head split: ALL ++// heads attend over the logical [0,n_kv) key range, decomposed by key POSITION into ++// a device-resident window (k_win/v_win, the bulk) + a pinned-host overflow tail ++// (k_tail/v_tail). The single two-region streaming op keeps the window FA device- ++// resident and DMAs only the tail, merging both through the online-softmax combine — ++// eliminating M2's O(n_kv) stream-back. k_tail==nullptr (n_kv<=window_cells) collapses ++// to a single resident region, byte-identical to vanilla build_attn_mha's FA branch. ++ggml_tensor * llm_graph_context::build_attn_mha_position_window( ++ ggml_tensor * q, ++ ggml_tensor * k_win, ++ ggml_tensor * v_win, ++ ggml_tensor * k_tail, ++ ggml_tensor * v_tail, ++ ggml_tensor * kq_b, ++ ggml_tensor * kq_mask, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il, ++ bool cpu_fa_tail) const { ++ GGML_ASSERT(cparams.flash_attn && "POSITION_WINDOW tactic requires flash-attention"); ++ GGML_ASSERT(kq_b == nullptr && "position window does not support KQ bias"); ++ GGML_ASSERT(v_mla == nullptr && "position window does not support MLA"); ++ GGML_ASSERT(sinks == nullptr && "position window does not support attention sinks"); ++ GGML_ASSERT(k_win && v_win); ++ ++ // Non-transposed V (the window-mode storage gate enforces !v_trans). ++ GGML_ASSERT(k_win->nb[1] <= k_win->nb[2] && "position window requires non-transposed V"); ++ ++ const auto n_stream = k_win->ne[3]; ++ ++ // Q: split streams then permute to FA layout (mirrors build_attn_mha exactly). ++ q = ggml_view_4d(ctx0, q, q->ne[0], q->ne[1], q->ne[2]/n_stream, n_stream, ++ q->nb[1], q->nb[2], q->nb[3]/n_stream, 0); ++ q = ggml_permute(ctx0, q, 0, 2, 1, 3); ++ ++ // Window (device bulk) → FA layout [d_head, n_win, n_head_kv, n_stream]. ++ ggml_tensor * k_win_p = ggml_permute(ctx0, k_win, 0, 2, 1, 3); ++ ggml_tensor * v_win_p = ggml_permute(ctx0, v_win, 0, 2, 1, 3); ++ // F32→F16 only (embedding non-causal path); quant K/V is dequantized on-lift ++ // inside the streaming op, so do NOT materialise an f16 copy here. ++ if (k_win_p->type == GGML_TYPE_F32) k_win_p = ggml_cast(ctx0, k_win_p, GGML_TYPE_F16); ++ if (v_win_p->type == GGML_TYPE_F32) v_win_p = ggml_cast(ctx0, v_win_p, GGML_TYPE_F16); ++ ++ // Tail (host overflow) → FA layout, or null when n_kv <= window_cells (resident). ++ ggml_tensor * k_tail_p = nullptr; ++ ggml_tensor * v_tail_p = nullptr; ++ if (k_tail) { ++ GGML_ASSERT(v_tail && "tail K present without tail V"); ++ k_tail_p = ggml_permute(ctx0, k_tail, 0, 2, 1, 3); ++ v_tail_p = ggml_permute(ctx0, v_tail, 0, 2, 1, 3); ++ if (k_tail_p->type == GGML_TYPE_F32) k_tail_p = ggml_cast(ctx0, k_tail_p, GGML_TYPE_F16); ++ if (v_tail_p->type == GGML_TYPE_F32) v_tail_p = ggml_cast(ctx0, v_tail_p, GGML_TYPE_F16); ++ } ++ ++ const float alibi_bias = hparams.f_max_alibi_bias; ++ const float logit_cap = hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f; ++ ++ ggml_tensor * cur; ++ // opencoti F5 M7 Stage 3d-S3 (#353): CPU_FA_TAIL. The GPU FAs the device-resident ++ // window [0,wc); the CPU FAs the pinned-host tail [wc,n_kv) (no DMA of the tail ++ // K/V) and emits a tiny [2+DV, n_head] partial [M,S,VKQ]; the window streaming op ++ // (partial mode, op_params[4]=1, src[5]=partial) converts + merges it through the ++ // online-softmax combine. Only the partial crosses PCIe — the win on slow links. ++ // The tail mask is the [wc,n_kv) column slice of kq_mask, made contiguous (the ++ // tail-partial op asserts a contiguous mask). Gated upstream on decode (n_q==1) + ++ // a real tail; here k_tail_p is guaranteed non-null when cpu_fa_tail is set. ++ if (cpu_fa_tail && k_tail_p) { ++ // bug-360: window cells live on the accessor's ne[2] ([n_embd_head, n_head_kv, ++ // n_win, n_stream] — get_k_window:3037), NOT ne[1] (= n_head_kv). The old ++ // `k_win->ne[1]` sliced m_tail at offset n_head_kv (=2 on Gemma global) instead ++ // of n_win; decode-inert (uniform-zero causal mask) but wrong for prefill / any ++ // non-uniform mask. The window key count after permute is k_win_p->ne[1] == ne[2]. ++ const int64_t wc = k_win->ne[2]; // window cells (key/position axis, pre-permute) ++ const int64_t n_kv_m = kq_mask->ne[0]; // full logical key span ++ ggml_tensor * m_tail = ggml_view_2d(ctx0, kq_mask, n_kv_m - wc, kq_mask->ne[1], ++ kq_mask->nb[1], (size_t) wc * kq_mask->nb[0]); ++ m_tail = ggml_cont(ctx0, m_tail); // tail-partial op needs a contiguous mask ++ ggml_tensor * partial = ggml_flash_attn_ext_tail_partial( ++ ctx0, q, k_tail_p, v_tail_p, m_tail, kq_scale, alibi_bias, logit_cap); ++ cur = ggml_streaming_flash_attn_window( ++ ctx0, q, k_win_p, v_win_p, nullptr, nullptr, kq_mask, kq_scale, alibi_bias, logit_cap); ++ // op_params[0..2] = {scale,max_bias,softcap} (set by the ctor); [3..] zero-init. ++ // Slot 4 == 1 marks CPU_FA_TAIL partial mode for ggml_cuda_streaming_flash_attn. ++ cur->op_params[4] = 1; ++ cur->src[5] = partial; // the [2+DV, n_head] CPU partial ++ } else { ++ cur = ggml_streaming_flash_attn_window( ++ ctx0, q, k_win_p, v_win_p, k_tail_p, v_tail_p, kq_mask, kq_scale, alibi_bias, logit_cap); ++ } ++ cb(cur, LLAMA_TENSOR_NAME_FATTN, il); ++ ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); ++ ++ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]); ++ return cur; ++} ++ + // opencoti F5 M3 neo — head-axis-split attention. Builds two flash_attn_ext + // ops on disjoint head ranges: + // GPU side: q[..., :gpu_heads*gqa, ...] × k_g × v_g (auto-routes to GPU +@@ -2409,7 +2613,23 @@ ggml_tensor * llm_graph_context::build_attn( + // null (build_attn for KV asserts this above at :2102), kq_b too; + // the M2 gate ensures non-transposed V. See docs/features/advanced_kv.md. + ggml_tensor * cur; +- if (cparams.neo_pipeline_mode != 0 && mctx_cur->headinfer_split_active(il)) { ++ if (mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::GPU_STREAM) { ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304), S1. GPU_STREAM ++ // rides the M2 head-split: the device head-group runs a resident FA, ++ // the host (spilled) head-group runs the streaming-FA op, and the two ++ // are concatenated on the head axis — same structure as the NEO branch ++ // below, but the host half uses GGML_OP_STREAMING_FLASH_ATTN (which S2 ++ // tiles over the key dimension with op-owned async DMA). Fetches the ++ // SPLIT accessors, not get_k/get_v. Assigned only when the GPU_STREAM ++ // spill tactic is selected and the layer did not fit resident. ++ ggml_tensor * k_g = mctx_cur->get_k_gpu(ctx0, il); ++ ggml_tensor * v_g = mctx_cur->get_v_gpu(ctx0, il); ++ ggml_tensor * k_c = mctx_cur->get_k_cpu(ctx0, il); ++ ggml_tensor * v_c = mctx_cur->get_v_cpu(ctx0, il); ++ cur = build_attn_mha_streaming(q, k_g, v_g, k_c, v_c, ++ mctx_cur->headinfer_gpu_heads(il), ++ kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ } else if (cparams.neo_pipeline_mode != 0 && mctx_cur->headinfer_split_active(il)) { + ggml_tensor * k_g = mctx_cur->get_k_gpu(ctx0, il); + ggml_tensor * v_g = mctx_cur->get_v_gpu(ctx0, il); + ggml_tensor * k_c = mctx_cur->get_k_cpu(ctx0, il); +@@ -2597,10 +2817,40 @@ ggml_tensor * llm_graph_context::build_attn( + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + + ggml_tensor * q = q_cur; +- ggml_tensor * k = mctx_cur->get_k(ctx0, il); +- ggml_tensor * v = mctx_cur->get_v(ctx0, il); + +- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ // opencoti-hook: f5-rolling-kv — M7 Stage 3c-5 (#348) POSITION_WINDOW dispatch on ++ // the iSWA path. Gemma-4's GLOBAL (!is_swa) overflowing layers carry a position ++ // window (device bulk [0,wc) + pinned-host tail [wc,n_kv)); route them through ++ // build_attn_mha_position_window, which keeps the window FA device-resident and ++ // streams only the tail via the two-region streaming op + online-softmax combine ++ // (the Stage-3a dst_lse keystone), eliminating M2's O(n_kv) stream-back. SWA ++ // layers and any MLA/KQ-bias/sinks path fall through to vanilla build_attn_mha. ++ const bool window_layer = ++ !is_swa && cparams.flash_attn && kq_b == nullptr && v_mla == nullptr && sinks == nullptr && ++ mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::POSITION_WINDOW && ++ mctx_cur->headinfer_window_active(il); ++ ++ ggml_tensor * cur; ++ if (window_layer) { ++ ggml_tensor * k_win = mctx_cur->get_k_window(ctx0, il); ++ ggml_tensor * v_win = mctx_cur->get_v_window(ctx0, il); ++ ggml_tensor * k_tail = mctx_cur->get_k_tail (ctx0, il); // nullptr when n_kv <= wc ++ ggml_tensor * v_tail = mctx_cur->get_v_tail (ctx0, il); ++ // opencoti F5 M7 Stage 3d-S3 (#353): CPU_FA_TAIL. When the S2 selector chose ++ // the CPU-FA tail tactic (slow PCIe link) AND there is a real overflow tail ++ // AND this is a single-token decode step (q->ne[2]==1 — the partial's per-row ++ // head-fastest order matches the combine only at n_q==1), FA the host tail on ++ // the CPU and merge a tiny partial instead of DMAing the tail. Otherwise (fast ++ // link, prefill, or no overflow) the unchanged two-region stream path runs. ++ const bool cpu_fa_tail = ++ (k_tail != nullptr) && (q->ne[2] == 1) && mctx_cur->headinfer_tail_is_cpu_fa(); ++ cur = build_attn_mha_position_window(q, k_win, v_win, k_tail, v_tail, ++ kq_b, kq_mask, sinks, v_mla, kq_scale, il, cpu_fa_tail); ++ } else { ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); ++ } + cb(cur, "kqv_out", il); + + if (v_rot) { +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -929,6 +929,57 @@ struct llm_graph_context { + float kq_scale, + int il) const; + ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304), S1. GPU_STREAM tactic. ++ // Structurally identical to build_attn_mha_neo (device head-group resident ++ // FA + host head-group FA + head-axis concat), but the host (spilled) ++ // head-group runs through the custom GGML_OP_STREAMING_FLASH_ATTN op ++ // instead of a plain flash_attn_ext. At S1 that op's CUDA forward is still ++ // ggml_cuda_flash_attn_ext verbatim, so the output is byte-identical to the ++ // NEO path (minus NEO's stream-pair registration, which only affects ++ // scheduling) — the S1 gate. S2 makes the op tile the host head-group's KEY ++ // dimension into a device slot pool with a cross-tile online-softmax, so ++ // the spilled KV need not fit one resident slot; S3 adds the copy/compute ++ // double-buffer. The host head-group is already pinned-host (the M2 split ++ // allocates k_cpu/v_cpu with the pinned cpu_buft), so streaming rides the ++ // M2 residency — no whole-layer host reorder. Flash-attention only (asserts ++ // cparams.flash_attn, kq_b == nullptr, v_mla == nullptr, gpu_heads > 0). ++ ggml_tensor * build_attn_mha_streaming( ++ ggml_tensor * q, // [n_embd_head_q, n_head_q, n_tokens] ++ ggml_tensor * k_g, // GPU-resident K subset (gpu_heads on dim 1) ++ ggml_tensor * v_g, // GPU-resident V subset (gpu_heads on dim 1) ++ ggml_tensor * k_c, // host-resident K subset (cpu_heads on dim 1) — streamed ++ ggml_tensor * v_c, // host-resident V subset (cpu_heads on dim 1) — streamed ++ uint32_t gpu_heads, ++ ggml_tensor * kq_b, // must be nullptr ++ ggml_tensor * kq_mask, ++ ggml_tensor * sinks, // [n_head_q] or nullptr ++ ggml_tensor * v_mla, // must be nullptr ++ float kq_scale, ++ int il) const; ++ ++ // opencoti F5 M7 Stage 3c-5 — POSITION_WINDOW tactic. The KV is split by KEY ++ // POSITION (not head): a device-resident window [0,wc) holds the bulk, a ++ // pinned-host tail [wc,n_kv) holds the overflow. ALL heads attend over the ++ // logical [0,n_kv) range via the two-region streaming op ++ // (ggml_streaming_flash_attn_window): the window FA stays device-resident and ++ // only the small tail is DMA'd, then both merge through the online-softmax ++ // combine — NO M2-style O(n_kv) stream-back. k_tail==nullptr (n_kv<=wc) collapses ++ // to a single resident FA, byte-identical to vanilla. Flash-attention only ++ // (asserts cparams.flash_attn, kq_b == nullptr, v_mla == nullptr, sinks == nullptr). ++ ggml_tensor * build_attn_mha_position_window( ++ ggml_tensor * q, // [n_embd_head_q, n_head_q, n_tokens] ++ ggml_tensor * k_win, // device window K (all heads, [0,wc) on dim 2) ++ ggml_tensor * v_win, // device window V ++ ggml_tensor * k_tail, // host tail K (all heads, [wc,n_kv) on dim 2) or nullptr ++ ggml_tensor * v_tail, // host tail V or nullptr ++ ggml_tensor * kq_b, // must be nullptr ++ ggml_tensor * kq_mask, // full [n_kv] logical mask ++ ggml_tensor * sinks, // must be nullptr ++ ggml_tensor * v_mla, // must be nullptr ++ float kq_scale, ++ int il, ++ bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) ++ + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -23,6 +23,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + uint32_t kv_size_initial, + uint32_t slot_shrink_idle_ms, + float headinfer_gpu_heads_frac, ++ uint32_t vram_target_mib, ++ float pcie_bw_gbps, ++ uint32_t kv_residency_mode, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +@@ -65,7 +68,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + kv_base = std::make_unique( + model, type_k, type_v, + v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, +- headinfer_gpu_heads_frac, ++ headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); +@@ -73,7 +76,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + kv_swa = std::make_unique( + model, type_k, type_v, + v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, +- headinfer_gpu_heads_frac, ++ headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse); + } + +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -30,6 +30,17 @@ public: + uint32_t slot_shrink_idle_ms, + // opencoti F5 M2 headinfer — forwarded to both kv_base and kv_swa. + float headinfer_gpu_heads_frac, ++ // opencoti F5 M7 Rolling KV — Rung 0 — forwarded to both kv_base ++ // and kv_swa. VRAM budget cap (MiB); 0 = maximize. Consulted only ++ // when headinfer_gpu_heads_frac is negative (auto). ++ uint32_t vram_target_mib, ++ // opencoti F5 M7 Rolling KV — Rung 2 (Stage 3d-S2 #352). Effective ++ // host->device PCIe bandwidth (GB/s); forwarded to both kv_base and ++ // kv_swa (only the overflowing base cache uses it). 0 = unknown. ++ float pcie_bw_gbps, ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob; ++ // forwarded to both kv_base and kv_swa. 0 = auto, 1 = head, 2 = window. ++ uint32_t kv_residency_mode, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -100,6 +100,218 @@ static ggml_tensor * ggml_mul_mat_aux( + // llama_kv_cache + // + ++// opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md ++// Sentinel for headinfer_gpu_heads_frac meaning "decide automatically": when the ++// incoming fraction is < 0, the KV-cache constructor computes an effective frac ++// from the offload device's free VRAM vs this cache's KV demand. eff_frac == 1.0 ++// (KV fits the budget) allocates NO split tensors — byte-identical to vanilla ++// (GPU_RESIDENT). Only a budget shortfall yields eff_frac < 1.0, which engages ++// the shipped M2 CPU-spill split, sized to fit. The non-negative path is the ++// manual escape hatch / upstream default and is used verbatim. ++static constexpr float OPENCOTI_HEADINFER_FRAC_AUTO = -1.0f; ++ ++// Conservative VRAM held back from the KV budget for compute scratch + cuBLAS ++// workspace, which are allocated AFTER the KV cache (graph_reserve / context ++// init). Model weights are already resident at KV-cache-construction time, so ++// the free-VRAM read here already nets them out; this reserve only covers the ++// not-yet-allocated compute buffers. --vram-target gives the operator a tighter ++// knob when this default is wrong for a given workload. ++static constexpr size_t OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES = (size_t) 1536 * 1024 * 1024; ++ ++// opencoti F5 M7 Rolling KV — Rung 0. Compute the effective GPU-head fraction ++// for HeadInfer (M2) from a VRAM-fit check. Returns 1.0 (GPU_RESIDENT — vanilla ++// fast path, no CPU split) when this cache's full KV demand fits the offload ++// device's budget; otherwise the largest fraction that fits, so the M2 split is ++// sized to exactly relieve the shortfall. Respects the same has_kv()/filter() ++// gating as the allocation loop so base/SWA caches each size only their layers. ++// Single-offload-device assumption: the budget is read from the first KV layer's ++// device; if KV layers span multiple devices the result is approximate (a ++// warning is logged) — later M7 rungs add a per-layer dynamic scheduler. ++static float opencoti_compute_auto_gpu_heads_frac( ++ const llama_model & model, ++ const llama_hparams & hparams, ++ ggml_type type_k, ++ ggml_type type_v, ++ bool v_trans, ++ bool offload, ++ uint32_t kv_size, ++ uint32_t n_stream, ++ uint32_t vram_target_mib, ++ const std::function & filter) { ++ // No GPU offload → nothing is "GPU-resident" and nothing to spill; the M2 ++ // split is only meaningful for offloaded layers. Stay at vanilla. ++ if (!offload) { ++ return 1.0f; ++ } ++ ++ size_t kv_demand = 0; ++ ggml_backend_dev_t dev = nullptr; ++ ggml_backend_dev_t dev_prev = nullptr; ++ int n_dev_distinct = 0; ++ for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ if (!hparams.has_kv(il)) { ++ continue; ++ } ++ if (filter && !filter(il)) { ++ continue; ++ } ++ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); ++ const uint32_t n_embd_v_gqa = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max(); ++ kv_demand += (size_t) ggml_row_size(type_k, n_embd_k_gqa) * kv_size; ++ kv_demand += (size_t) ggml_row_size(type_v, n_embd_v_gqa) * kv_size; ++ ++ ggml_backend_dev_t d = model.dev_layer(il); ++ if (!dev) { ++ dev = d; ++ } ++ if (d != dev_prev) { ++ dev_prev = d; ++ ++n_dev_distinct; ++ } ++ } ++ kv_demand *= (size_t) n_stream; ++ ++ if (kv_demand == 0 || !dev) { ++ return 1.0f; ++ } ++ if (n_dev_distinct > 1) { ++ LLAMA_LOG_WARN("%s: KV layers span %d devices; auto headinfer fraction is " ++ "approximate (budget read from the first device)\n", __func__, n_dev_distinct); ++ } ++ ++ size_t free_vram = 0, total_vram = 0; ++ ggml_backend_dev_memory(dev, &free_vram, &total_vram); ++ ++ size_t budget = free_vram; ++ if (vram_target_mib > 0) { ++ budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); ++ } ++ budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ : 0; ++ ++ if (kv_demand <= budget) { ++ LLAMA_LOG_INFO("%s: auto residency = GPU_RESIDENT (KV %.0f MiB fits budget " ++ "%.0f MiB of %.0f MiB free); headinfer split OFF\n", __func__, ++ kv_demand / 1048576.0, budget / 1048576.0, free_vram / 1048576.0); ++ return 1.0f; ++ } ++ ++ float frac = (float) ((double) budget / (double) kv_demand); ++ frac = std::min(frac, 0.99f); ++ frac = std::max(frac, 0.0f); ++ LLAMA_LOG_INFO("%s: auto residency = CPU_SPILL (KV %.0f MiB exceeds budget " ++ "%.0f MiB of %.0f MiB free); headinfer frac = %.3f\n", __func__, ++ kv_demand / 1048576.0, budget / 1048576.0, free_vram / 1048576.0, (double) frac); ++ return frac; ++} ++ ++// opencoti F5 M7 Rolling KV — Stage 3c. Position-axis residency sizing: how many KV ++// *cells* (positions) [0, C) fit resident in VRAM as the device "window"; the overflow ++// [C, kv_size) becomes the pinned-host "tail" streamed device-ward per tile. Mirrors ++// opencoti_compute_auto_gpu_heads_frac's budget math but on the POSITION axis (head- ++// agnostic), so it sidesteps Gemma-4's 2-KV-head binary spill (the head axis can only ++// express 0/50/100%). Returns kv_size — meaning "no tail", fully resident, byte-identical ++// to vanilla — when the full cache fits the budget; otherwise the largest cell count C ++// that fits, rounded DOWN to n_pad (covers q8_0 blck alignment, bug-225 class) and clamped ++// to [n_pad, kv_size - n_pad]. Single-offload-device assumption mirrors the auto-frac sizer. ++// ++// BELT-AND-SUSPENDERS (bug-263, user-requested "GO + belt-and-suspenders"): the decode ++// window FA processes exactly C keys, and C in the band (480, 512] maps to ntiles_KV==16 → ++// gridDim.x == WARP_SIZE for head_dim > 256 — the one uniform tiling that maximizes online- ++// softmax combine-order divergence (numerically valid but greedy-fragile; bug-263 is that ++// sensitivity, not a defect). We snap C below that band so the resident window never sits on ++// it. Universally harmless (≤ 32-cell shrink in a rare tiny-budget edge case); the symmetric ++// guard for tail *tiles* lives in the streaming forward. See .wolf/buglog.json bug-263. ++static uint32_t opencoti_compute_resident_window_cells( ++ const llama_model & model, ++ const llama_hparams & hparams, ++ ggml_type type_k, ++ ggml_type type_v, ++ bool v_trans, ++ bool offload, ++ uint32_t kv_size, ++ uint32_t n_stream, ++ uint32_t vram_target_mib, ++ uint32_t n_pad, ++ const std::function & filter) { ++ if (!offload || kv_size == 0) { ++ return kv_size; // no position-axis spill without GPU offload ++ } ++ ++ size_t per_cell = 0; // bytes for ONE position cell across all this cache's KV layers ++ ggml_backend_dev_t dev = nullptr; ++ for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ if (!hparams.has_kv(il)) { ++ continue; ++ } ++ if (filter && !filter(il)) { ++ continue; ++ } ++ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); ++ const uint32_t n_embd_v_gqa = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max(); ++ per_cell += (size_t) ggml_row_size(type_k, n_embd_k_gqa); ++ per_cell += (size_t) ggml_row_size(type_v, n_embd_v_gqa); ++ if (!dev) { ++ dev = model.dev_layer(il); ++ } ++ } ++ per_cell *= (size_t) n_stream; ++ if (per_cell == 0 || !dev) { ++ return kv_size; ++ } ++ ++ size_t free_vram = 0, total_vram = 0; ++ ggml_backend_dev_memory(dev, &free_vram, &total_vram); ++ size_t budget = free_vram; ++ if (vram_target_mib > 0) { ++ budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); ++ } ++ budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ : 0; ++ ++ if (per_cell * (size_t) kv_size <= budget) { ++ LLAMA_LOG_INFO("%s: position window = FULLY RESIDENT (KV %.0f MiB fits budget " ++ "%.0f MiB of %.0f MiB free); no host tail\n", __func__, ++ per_cell * (size_t) kv_size / 1048576.0, budget / 1048576.0, free_vram / 1048576.0); ++ return kv_size; ++ } ++ ++ // 3c-5 / bug-267: the window split boundary wc feeds per-tile flash-attention ++ // DIRECTLY — window keys [0,wc) and tail keys [wc,n_kv) each become their own ++ // per-tile FA sub-calls in ggml_cuda_streaming_flash_attn. For head_dim 512/576 ++ // (Gemma global) the CUDA FA kernel HARD-REQUIRES gqa_opt, i.e. ++ // K->ne[1] % FATTN_KQ_STRIDE(256) == 0 (fattn.cu get_best_fattn_kernel); a wc that ++ // is merely n_pad(32)-aligned makes the window's (and tail's) partial last tile ++ // resolve to BEST_FATTN_KERNEL_NONE → GGML_ABORT("fatal error") on the warmup run. ++ // n_kv is already padded to a 256-multiple by the FA graph, so aligning wc to 256 ++ // keeps BOTH regions 256-aligned (n_tail = n_kv - wc inherits it). 256 is a ++ // multiple of the q8_0 block (32, bug-225) and of every n_pad we use, so this ++ // strictly TIGHTENS the prior alignment. Align to lcm(n_pad, 256) (= max for the ++ // power-of-2 n_pad llama.cpp uses). The bug-263 (480,512] band guard is dropped: ++ // bug-263 is RESOLVED as FP combine-order sensitivity (acceptance gate is ++ // cosine+semantic, never greedy byte-equality), and 480 is not 256-aligned anyway. ++ const uint32_t pad = n_pad ? n_pad : 32; ++ const uint32_t fa_align = 256; // FATTN_KQ_STRIDE ++ const uint32_t walign = pad >= fa_align ? pad : fa_align; // lcm(pad,256), power-of-2 pad ++ uint32_t cells = (uint32_t) (budget / per_cell); ++ cells = (cells / walign) * walign; // round DOWN to the FA-aligned window ++ if (cells < walign) { ++ cells = walign; // never below one FA stride (~0.5 MiB — never OOMs) ++ } ++ if (kv_size > walign && cells > kv_size - walign) { ++ cells = ((kv_size - walign) / walign) * walign; // keep >= one FA stride of tail, stay aligned ++ if (cells < walign) { ++ cells = walign; ++ } ++ } ++ LLAMA_LOG_INFO("%s: position window = %u / %u cells resident (%.0f MiB budget, " ++ "%.0f MiB/cell-set); host tail = %u cells\n", __func__, ++ cells, kv_size, budget / 1048576.0, per_cell / 1048576.0, kv_size - cells); ++ return cells; ++} ++ + llama_kv_cache::llama_kv_cache( + const llama_model & model, + ggml_type type_k, +@@ -111,6 +323,9 @@ llama_kv_cache::llama_kv_cache( + uint32_t kv_size_initial, + uint32_t slot_shrink_idle_ms, + float headinfer_gpu_heads_frac, ++ uint32_t vram_target_mib, ++ float pcie_bw_gbps, ++ uint32_t kv_residency_mode, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -154,6 +369,60 @@ llama_kv_cache::llama_kv_cache( + + const uint32_t n_layer_kv = hparams.n_layer_kv(); + ++ // opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md ++ // Resolve the effective HeadInfer fraction once for this cache. A negative ++ // incoming frac (OPENCOTI_HEADINFER_FRAC_AUTO) means "decide from free VRAM": ++ // KV fits → 1.0 (GPU_RESIDENT, byte-identical to vanilla — no split tensors); ++ // shortfall → the largest fit-sized fraction (engages the M2 CPU-spill split). ++ // A non-negative frac is the manual escape hatch / upstream default, used ++ // verbatim. Every split decision below reads eff_gpu_heads_frac, not the raw ++ // ctor argument, so the sentinel never leaks into the allocation math. ++ const float eff_gpu_heads_frac = headinfer_gpu_heads_frac < 0.0f ++ ? opencoti_compute_auto_gpu_heads_frac(model, hparams, type_k, type_v, ++ v_trans, offload, kv_size, n_stream, vram_target_mib, filter) ++ : headinfer_gpu_heads_frac; ++ ++ // opencoti F5 M7 Stage 3c — position-axis window sizing. Computed + logged now so the ++ // sizing math can be validated against real models (boot at various --vram-target and ++ // read the logged C) BEFORE the storage/forward wiring lands in the next 3c sub-stages. ++ // window_cells == kv_size ⇒ no host tail (fully resident, byte-identical vanilla). The ++ // value is intentionally unused downstream at this sub-stage. See docs/features/advanced_kv.md. ++ const uint32_t window_cells = opencoti_compute_resident_window_cells( ++ model, hparams, type_k, type_v, v_trans, offload, kv_size, n_stream, ++ vram_target_mib, n_pad, filter); ++ // opencoti F5 M7 Stage 4 (#338) — position-window mode is driven by the ++ // --kv-residency-mode knob (cparams.kv_residency_mode): 0=auto, 1=head, 2=window. ++ // The ELIGIBILITY predicate is the validated POSITION_WINDOW class: the sizer found a ++ // real overflow (0 < C < kv_size) on an offloaded, non-transposed-V, APPEND-ONLY ++ // (swa_type==NONE) cache. window mode is MUTUALLY EXCLUSIVE with the M2 head split ++ // (forced off below), so the shared k_per_stream/k_cpu_per_stream slots carry the ++ // position split, never both. ++ // ++ // bug-268: window mode is valid ONLY on an APPEND-ONLY cache whose cell index equals ++ // the key position (the keystone — see plan). That holds for the iSWA base / non-SWA ++ // cache (swa_type == NONE, n_swa == 0, find_slot append-only) but NOT for the ++ // sliding-window cache, whose cells rotate/reuse (idx != position) — a tiny ++ // --vram-target made the sizer return C 0 && window_cells < kv_size ++ && swa_type == LLAMA_SWA_TYPE_NONE; ++ // auto (0): window when eligible. head (1): never window (force the M2 split). ++ // window (2): force window when eligible; warn + fall back when ineligible. ++ const bool window_mode = window_eligible && kv_residency_mode != 1; ++ if (kv_residency_mode == 2 && !window_eligible) { ++ LLAMA_LOG_WARN("%s: --kv-residency-mode window requested but this cache is ineligible " ++ "(offload=%d v_trans=%d window_cells=%u/%u swa_type=%d) — falling back to " ++ "head/resident\n", __func__, offload, v_trans, window_cells, kv_size, (int) swa_type); ++ } ++ if (window_mode) { ++ LLAMA_LOG_INFO("%s: rolling-kv POSITION_WINDOW mode ON (--kv-residency-mode %s) — " ++ "window %u / %u cells, host tail %u (M2 head split forced off)\n", __func__, ++ kv_residency_mode == 2 ? "window" : "auto", ++ window_cells, kv_size, kv_size - window_cells); ++ } ++ + // define a comparator for the buft -> ctx map to ensure that the order is well-defined: + struct ggml_backend_buft_comparator { + bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const { +@@ -273,6 +542,7 @@ llama_kv_cache::llama_kv_cache( + // pageable CPU buft when no CUDA dev / no dev_host_buffer_type. Pinned + // host buffers DMA at full PCIe bandwidth; pageable hits ~50%. + ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type(); ++ + if (offload) { + auto * dev = model.dev_layer(il); + buft = ggml_backend_dev_buffer_type(dev); +@@ -311,12 +581,15 @@ llama_kv_cache::llama_kv_cache( + uint32_t n_embd_v_gpu = n_embd_v_gqa; + uint32_t n_embd_k_cpu = 0; + uint32_t n_embd_v_cpu = 0; +- if (headinfer_gpu_heads_frac < 1.0f && offload && !v_trans) { ++ // opencoti F5 M7 Stage 3c — POSITION_WINDOW and the M2 head split are ++ // mutually exclusive: when window mode is on, force the head split off ++ // so k_per_stream carries the position window (ALL heads), never both. ++ if (!window_mode && eff_gpu_heads_frac < 1.0f && offload && !v_trans) { + const uint32_t n_head_kv = hparams.n_head_kv(il); + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); + const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); + if (n_head_kv > 1) { +- uint32_t g = (uint32_t) std::lround((double) headinfer_gpu_heads_frac * (double) n_head_kv); ++ uint32_t g = (uint32_t) std::lround((double) eff_gpu_heads_frac * (double) n_head_kv); + g = std::max(1u, std::min(g, n_head_kv - 1)); + gpu_heads = g; + n_embd_k_gpu = g * n_embd_head_k; +@@ -328,6 +601,14 @@ llama_kv_cache::llama_kv_cache( + } + } + ++ // opencoti F5 M7 Stage 3c — position-window cell counts. In window ++ // mode the device tensor spans only the resident window [0, wc) and the ++ // host tail tensor spans [wc, kv_size) (tail_c cells); both carry ALL ++ // heads. Outside window mode wc == kv_size and tail_c == 0, so the ++ // tensor shapes below are byte-identical to the pre-3c layout. ++ const uint32_t wc = window_mode ? window_cells : kv_size; ++ const uint32_t tail_c = window_mode ? (kv_size - window_cells) : 0; ++ + // opencoti F4 M3 Phase 4 — per-stream tensor split. + // Each stream gets its own OWNING 3-D tensor of shape + // [n_embd, kv_size, 1]. Dim 3 == 1 keeps stride math +@@ -349,11 +630,13 @@ llama_kv_cache::llama_kv_cache( + throw std::runtime_error("failed to create ggml context for kv cache"); + } + ++ // opencoti F5 M7 Stage 3c — `wc` is kv_size outside window mode ++ // (byte-identical), or the resident window length in window mode. + ggml_tensor * k_s = has_k +- ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, kv_size, 1) ++ ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, wc, 1) + : nullptr; + ggml_tensor * v_s = has_v +- ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, kv_size, 1) ++ ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, wc, 1) + : nullptr; + + // Tensor naming preserves the pre-Phase-4 name in unified +@@ -382,34 +665,44 @@ llama_kv_cache::llama_kv_cache( + // in its own host buffer (Phase 4-B). Vectors stay empty + // when gpu_heads == 0 so headinfer_split_active() can test + // by vector size. +- if (gpu_heads > 0) { +- // opencoti F5 M7-A (0034) — was ggml_backend_cpu_buffer_type() +- // (pageable). Now uses the layer-scope cpu_buft resolved above: +- // pinned host buffer when CUDA is the layer dev, plain CPU buft +- // otherwise. Pinned unlocks full PCIe DMA for M2's CPU-half +- // stream-back AND is the foundation Rolling KV (M7) builds on. ++ // opencoti F5 M7-A (0034) — host-resident KV subset. M2 head split ++ // (gpu_heads > 0) stores the CPU head-half here; F5 M7 Stage 3c ++ // window mode (window_mode) instead stores the ALL-heads position ++ // tail [wc, kv_size) here. The two are mutually exclusive (window ++ // mode forces gpu_heads == 0), so exactly one branch sizes the ++ // tensor. The host buffer is the pinned cpu_buft when CUDA is the ++ // layer dev — full PCIe DMA for the stream-back. ++ if (gpu_heads > 0 || window_mode) { + ggml_context * ctx_cpu_s = ctx_for_buft(cpu_buft, s); + if (!ctx_cpu_s) { + throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); + } ++ // window mode: ALL heads, tail_c cells; M2: CPU head subset, kv_size cells. ++ const uint32_t host_k_embd = window_mode ? n_embd_k_gqa : n_embd_k_cpu; ++ const uint32_t host_v_embd = window_mode ? n_embd_v_gqa : n_embd_v_cpu; ++ const uint32_t host_cells = window_mode ? tail_c : kv_size; + ggml_tensor * k_cpu_s = has_k +- ? ggml_new_tensor_3d(ctx_cpu_s, type_k, n_embd_k_cpu, kv_size, 1) ++ ? ggml_new_tensor_3d(ctx_cpu_s, type_k, host_k_embd, host_cells, 1) + : nullptr; + ggml_tensor * v_cpu_s = has_v +- ? ggml_new_tensor_3d(ctx_cpu_s, type_v, n_embd_v_cpu, kv_size, 1) ++ ? ggml_new_tensor_3d(ctx_cpu_s, type_v, host_v_embd, host_cells, 1) + : nullptr; ++ // Name distinguishes the two layouts for state-IO grep: ++ // "cache_k_tail" (position tail) vs "cache_k_cpu" (head half). ++ const char * k_base = window_mode ? "cache_k_tail" : "cache_k_cpu"; ++ const char * v_base = window_mode ? "cache_v_tail" : "cache_v_cpu"; + if (k_cpu_s) { + if (n_stream == 1) { +- ggml_format_name(k_cpu_s, "cache_k_cpu_l%d", il); ++ ggml_format_name(k_cpu_s, "%s_l%d", k_base, il); + } else { +- ggml_format_name(k_cpu_s, "cache_k_cpu_l%d_s%u", il, s); ++ ggml_format_name(k_cpu_s, "%s_l%d_s%u", k_base, il, s); + } + } + if (v_cpu_s) { + if (n_stream == 1) { +- ggml_format_name(v_cpu_s, "cache_v_cpu_l%d", il); ++ ggml_format_name(v_cpu_s, "%s_l%d", v_base, il); + } else { +- ggml_format_name(v_cpu_s, "cache_v_cpu_l%d_s%u", il, s); ++ ggml_format_name(v_cpu_s, "%s_l%d_s%u", v_base, il, s); + } + } + k_cpu_per_stream.push_back(k_cpu_s); +@@ -429,11 +722,13 @@ llama_kv_cache::llama_kv_cache( + // are byte-identical to pre-Phase-4 because the view has + // identical ne/nb and (in unified mode) the same underlying + // buffer layout. ++ // opencoti F5 M7 Stage 3c — the view spans the device tensor's ++ // actual ne[1] (== wc), which is kv_size outside window mode. + k_stream.push_back(k_s +- ? ggml_view_2d(ctx_s, k_s, n_embd_k_gpu, kv_size, k_s->nb[1], 0) ++ ? ggml_view_2d(ctx_s, k_s, n_embd_k_gpu, wc, k_s->nb[1], 0) + : nullptr); + v_stream.push_back(v_s +- ? ggml_view_2d(ctx_s, v_s, n_embd_v_gpu, kv_size, v_s->nb[1], 0) ++ ? ggml_view_2d(ctx_s, v_s, n_embd_v_gpu, wc, v_s->nb[1], 0) + : nullptr); + } + +@@ -445,6 +740,7 @@ llama_kv_cache::llama_kv_cache( + k_stream, v_stream, + k_cpu_per_stream, v_cpu_per_stream, + gpu_heads, ++ window_mode ? window_cells : 0u, + }); + } + +@@ -472,6 +768,12 @@ llama_kv_cache::llama_kv_cache( + } + } + ++ // opencoti F5 M7 Rolling KV — Rung 1. Derive the per-layer tactic table + ++ // streaming plan from the split decisions just baked into `layers`, and log ++ // the histogram. eff_bw (Stage 3d-S2 #352) feeds the overflow tail-tactic ++ // crossover selector. Compute-inert at Rung 1 (nothing reads the table yet). ++ build_rolling_kv_plan(pcie_bw_gbps); ++ + // opencoti F4 M3 Phase 1 — see docs/decisions/0001-lazy-slot-context.md + // Eager malloc, deferred zero-fill. The address-space reservation done + // by ggml_backend_alloc_ctx_tensors_from_buft is essentially free on +@@ -616,35 +918,41 @@ void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) { + // tensor. M2's CPU heads follow the same pattern via + // k_cpu_per_stream / v_cpu_per_stream (empty when no split). + for (const auto & layer : layers) { ++ // opencoti F5 M7 Stage 3c — in window mode the device tensor holds only ++ // the resident window cells [0, wc) and the host tensor holds the tail ++ // cells [wc, kv_size) at local index (cell - wc). Clamp every memset to ++ // the destination tensor's own cell capacity (ne[1]) so a partial-window ++ // tensor is never written past its end (the warmup zero-fill bug). wc == 0 ++ // ⇒ M2 / vanilla layout (host tensor shares the device cell range), and ++ // the clamp is a no-op (b <= kv_size == ne[1]) → byte-identical to before. ++ const uint32_t wc = layer.window_cells; ++ // Clear logical cells [a, b) of a tensor whose local cell 0 maps to ++ // logical cell `base`, clamped to the tensor's capacity (ne[1]). ++ auto clear_range = [&](ggml_tensor * t, uint32_t base) { ++ if (!t) { ++ return; ++ } ++ const uint32_t cap = (uint32_t) t->ne[1]; ++ const uint32_t lo0 = a > base ? a - base : 0; ++ const uint32_t hi0 = b > base ? b - base : 0; ++ const uint32_t lo = lo0 < cap ? lo0 : cap; ++ const uint32_t hi = hi0 < cap ? hi0 : cap; ++ if (hi > lo) { ++ ggml_backend_tensor_memset(t, 0, (size_t) lo * t->nb[1], (size_t) (hi - lo) * t->nb[1]); ++ } ++ }; + for (uint32_t st = 0; st < n_stream; ++st) { +- auto * k = layer.k_per_stream[st]; +- if (k) { +- const size_t off = (size_t) a * (size_t) k->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) k->nb[1]; +- ggml_backend_tensor_memset(k, 0, off, sz); +- } +- auto * v = layer.v_per_stream[st]; +- if (v) { +- const size_t off = (size_t) a * (size_t) v->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) v->nb[1]; +- ggml_backend_tensor_memset(v, 0, off, sz); +- } +- // M2 headinfer host-resident head subset (same [a, b) cells). ++ // Device window (or full) tensor: local cell 0 == logical cell 0. ++ clear_range(layer.k_per_stream[st], 0); ++ clear_range(layer.v_per_stream[st], 0); ++ // Host tensor: window tail's local cell 0 == logical cell wc; the M2 ++ // head subset (wc == 0) shares the device cell range (base 0). ++ const uint32_t host_base = wc; + if (st < layer.k_cpu_per_stream.size()) { +- auto * k_cpu = layer.k_cpu_per_stream[st]; +- if (k_cpu) { +- const size_t off = (size_t) a * (size_t) k_cpu->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) k_cpu->nb[1]; +- ggml_backend_tensor_memset(k_cpu, 0, off, sz); +- } ++ clear_range(layer.k_cpu_per_stream[st], host_base); + } + if (st < layer.v_cpu_per_stream.size()) { +- auto * v_cpu = layer.v_cpu_per_stream[st]; +- if (v_cpu) { +- const size_t off = (size_t) a * (size_t) v_cpu->nb[1]; +- const size_t sz = (size_t) (b - a) * (size_t) v_cpu->nb[1]; +- ggml_backend_tensor_memset(v_cpu, 0, off, sz); +- } ++ clear_range(layer.v_cpu_per_stream[st], host_base); + } + } + } +@@ -1910,6 +2218,51 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k + return result; + } + ++ // opencoti F5 M7 Stage 3c — position-window reassembly. The device tensor ++ // holds the resident window [0, wc); the pinned host tail holds [wc, kv_size). ++ // Build a window view over the device tensor (dim-2 length n_win) + a tail ++ // view over the host tensor (dim-2 length n_tail) and concat on the POSITION ++ // axis (dim 2). The backend scheduler streams the host tail to the compute ++ // backend — same mechanism as the M2 head concat above. When n_kv <= wc the ++ // tail is empty → a single window view, byte-identical to vanilla resident. ++ // This is the correctness fallback (3c-4); the perf path (separate window/tail ++ // FA + online-softmax combine, no streamback) is build_attn_mha_position_window. ++ if (layer.window_cells > 0 && !layer.k_cpu_per_stream.empty() && layer.k_cpu_per_stream[0]) { ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_k_gqa2 = (uint64_t) n_head_kv * n_embd_head_k; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_win = n_kv < wc ? n_kv : wc; ++ const uint32_t n_tail = n_kv > wc ? n_kv - wc : 0; ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * k_s = layer.k_per_stream[s_cache]; ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ ggml_tensor * win_v = ggml_view_4d(ctx, k_s, ++ n_embd_head_k, n_head_kv, n_win, 1, ++ ggml_row_size(k_s->type, n_embd_head_k), ++ ggml_row_size(k_s->type, n_embd_k_gqa2), ++ ggml_row_size(k_s->type, n_embd_k_gqa2 * (uint64_t) k_s->ne[1]), ++ 0); ++ if (n_tail == 0) { ++ return win_v; ++ } ++ ggml_tensor * tail_v = ggml_view_4d(ctx, kc_s, ++ n_embd_head_k, n_head_kv, n_tail, 1, ++ ggml_row_size(kc_s->type, n_embd_head_k), ++ ggml_row_size(kc_s->type, n_embd_k_gqa2), ++ ggml_row_size(kc_s->type, n_embd_k_gqa2 * (uint64_t) kc_s->ne[1]), ++ 0); ++ return ggml_concat(ctx, win_v, tail_v, 2); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++ } ++ + // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream axis + // (dim 3). In unified mode (sinfo.s1 == sinfo.s0 == 0) the loop does + // not fire, yielding a single 4-D view of layer.k_per_stream[0] — +@@ -1983,6 +2336,46 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k + return result; + } + ++ // opencoti F5 M7 Stage 3c — position-window reassembly (mirror of get_k). ++ // window mode is gated to !v_trans at construction, so V is always the ++ // non-transposed (heads in dim 1) layout here. Concat device window ⊕ host ++ // tail on the position axis (dim 2); empty tail (n_kv <= wc) → window only. ++ if (layer.window_cells > 0 && !layer.v_cpu_per_stream.empty() && layer.v_cpu_per_stream[0]) { ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_v_gqa2 = (uint64_t) n_head_kv * n_embd_head_v; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_win = n_kv < wc ? n_kv : wc; ++ const uint32_t n_tail = n_kv > wc ? n_kv - wc : 0; ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache]; ++ ggml_tensor * win_v = ggml_view_4d(ctx, v_s, ++ n_embd_head_v, n_head_kv, n_win, 1, ++ ggml_row_size(v_s->type, n_embd_head_v), ++ ggml_row_size(v_s->type, n_embd_v_gqa2), ++ ggml_row_size(v_s->type, n_embd_v_gqa2 * (uint64_t) v_s->ne[1]), ++ 0); ++ if (n_tail == 0) { ++ return win_v; ++ } ++ ggml_tensor * tail_v = ggml_view_4d(ctx, vc_s, ++ n_embd_head_v, n_head_kv, n_tail, 1, ++ ggml_row_size(vc_s->type, n_embd_head_v), ++ ggml_row_size(vc_s->type, n_embd_v_gqa2), ++ ggml_row_size(vc_s->type, n_embd_v_gqa2 * (uint64_t) vc_s->ne[1]), ++ 0); ++ return ggml_concat(ctx, win_v, tail_v, 2); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++ } ++ + // opencoti F4 M3 Phase 4-C: per-stream view + concat along stream + // axis. Same shape as get_k, but with v_trans branching. Unified mode + // (sinfo.s1 == sinfo.s0 == 0) keeps a single 4-D view → byte-identical. +@@ -2035,6 +2428,307 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k + // per-step O(n_kv) CPU→GPU stream-back. Dimensions and strides are identical + // to the corresponding halves inside M2's get_k/get_v. + ++// opencoti F5 M7 Stage 3d-S1 (#351) — compute microbench (auto-adapt loop step 2, ++// docs/features/rolling_kv.md). Measure THIS host's GPU flash-attention compute time ++// per KV cell so the Stage-3d-S2 tactic selector can size tile_bytes_max = ++// compute_ms * eff_bw_gbps * 0.8 and choose stream-vs-CPU-FA from MEASURED hardware ++// characteristics — never hardcoded (PCIe 3.0 x1 … 5.0 x16 all auto-adapt). ++// ++// Host-side: builds a synthetic decode-shape ggml_flash_attn_ext over `bench_cells` ++// keys and runs it on a transient backend over the model's device — the already-loaded ++// CUDA DSO dispatches the kernel, so NO DSO rebuild is needed. Mirrors the real ++// build_attn_mha FA contract (post-permute q[hd,n_q,n_head], k/v[hd,n_kv,n_head_kv], ++// F16 mask). Never aborts boot: every shortfall (no GPU, unsupported op, non-GQA ++// geometry, alloc miss) returns 0.0f and the selector keeps the stream default. ++static float opencoti_fa_compute_probe_ms( ++ const llama_model & model, const llama_hparams & hparams, ++ int32_t il, ggml_type type_k, ggml_type type_v, int64_t bench_cells, ++ bool on_cpu) { ++ // GPU probe (on_cpu=false): the model's compute device for layer il — the already-loaded ++ // CUDA DSO dispatches the FA kernel. CPU probe (on_cpu=true, Stage 3d-S2 #352 companion): ++ // the host CPU backend (ggml-cpu / iqk FA) so the selector can MODEL the no-DMA CPU-FA tail ++ // and choose stream-vs-CPU from measured hardware — never hardcoded. ++ ggml_backend_dev_t dev = on_cpu ++ ? ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU) ++ : model.dev_layer(il); ++ if (!dev) { ++ return 0.0f; ++ } ++ // NB: `ggml_backend_dev_type` names both the enum and the accessor function; in C++ ++ // `ggml_backend_dev_type(dev)` would parse as a functional cast, so read the type via props. ++ ggml_backend_dev_props dprops = {}; ++ ggml_backend_dev_get_props(dev, &dprops); ++ // Compare enumerator values directly — do NOT declare a variable of type ++ // `ggml_backend_dev_type` (it names both the enum and a function; the function ++ // hides the type in ordinary lookup, so such a declaration mis-parses). ++ const bool type_ok = on_cpu ++ ? (dprops.type == GGML_BACKEND_DEVICE_TYPE_CPU) ++ : (dprops.type == GGML_BACKEND_DEVICE_TYPE_GPU); ++ if (!type_ok) { ++ return 0.0f; // device-type mismatch; selector falls back (stream default / no CPU model) ++ } ++ ++ const int64_t hd_k = (int64_t) hparams.n_embd_head_k(il); ++ const int64_t hd_v = (int64_t) hparams.n_embd_head_v(il); ++ const int64_t n_head = (int64_t) hparams.n_head(il); ++ const int64_t n_head_kv = (int64_t) hparams.n_head_kv(il); ++ // GQA contract: ggml_flash_attn_ext build asserts can_mul_mat(k,q) ⇒ n_head % n_head_kv == 0. ++ if (hd_k <= 0 || hd_v <= 0 || n_head <= 0 || n_head_kv <= 0 || (n_head % n_head_kv) != 0 || bench_cells < 256) { ++ return 0.0f; ++ } ++ ++ ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); ++ if (!backend) { ++ return 0.0f; ++ } ++ ++ const size_t meta_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); ++ ggml_init_params ip = { meta_size, nullptr, /*.no_alloc =*/ true }; ++ ggml_context * ctx = ggml_init(ip); ++ if (!ctx) { ++ ggml_backend_free(backend); ++ return 0.0f; ++ } ++ ++ // Post-permute decode FA shapes (mirror build_attn_mha): head count lives in ne[2] ++ // (GQA broadcast q->ne[2] % k->ne[2] == 0); the kq_mask is F16 (the op asserts F16). ++ ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, hd_k, 1, n_head, 1); ++ ggml_tensor * k = ggml_new_tensor_4d(ctx, type_k, hd_k, bench_cells, n_head_kv, 1); ++ ggml_tensor * v = ggml_new_tensor_4d(ctx, type_v, hd_v, bench_cells, n_head_kv, 1); ++ ggml_tensor * mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, bench_cells, 1, 1, 1); ++ const float scale = 1.0f / sqrtf((float) hd_k); ++ ggml_tensor * fa = ggml_flash_attn_ext(ctx, q, k, v, mask, scale, 0.0f, 0.0f); ++ ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); ++ ++ if (!ggml_backend_dev_supports_op(dev, fa)) { ++ ggml_free(ctx); ++ ggml_backend_free(backend); ++ return 0.0f; ++ } ++ ++ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); ++ if (!buf) { ++ ggml_free(ctx); ++ ggml_backend_free(backend); ++ return 0.0f; ++ } ++ ggml_backend_buffer_clear(buf, 0); // zero inputs: mask 0 ⇒ attend-all, K/V 0 ⇒ finite output, no NaN ++ ++ ggml_cgraph * gf = ggml_new_graph(ctx); ++ ggml_build_forward_expand(gf, fa); ++ ++ // warmup (kernel selection / JIT / first-touch), then time a few iterations ++ ggml_backend_graph_compute(backend, gf); ++ ggml_backend_synchronize(backend); ++ ++ const int iters = 5; ++ const auto t0 = std::chrono::steady_clock::now(); ++ for (int i = 0; i < iters; ++i) { ++ ggml_backend_graph_compute(backend, gf); ++ } ++ ggml_backend_synchronize(backend); ++ const auto t1 = std::chrono::steady_clock::now(); ++ const double total_ms = std::chrono::duration(t1 - t0).count(); ++ ++ ggml_backend_buffer_free(buf); ++ ggml_free(ctx); ++ ggml_backend_free(backend); ++ ++ return (float) (total_ms / (double) iters); // ms for one FA over bench_cells keys ++} ++ ++// opencoti F5 M7 Rolling KV — Rung 1 (M7-B). Derive the per-layer tactic table ++// from the split decisions already baked into `layers` during construction, and ++// log the histogram. At Rung 1 the only tactics that appear are GPU_RESIDENT ++// (gpu_heads == 0, full KV on device, vanilla fast path) and CPU_SPILL ++// (gpu_heads > 0, the shipped M2 head-split). The streaming tactics ++// (GPU_STREAM / GPU_STREAM_SUBTILE) + the tile-sizing fields (eff_bw_gbps from ++// W1 pcie_profile threaded via cparams, compute_ms from a live FA microbench, ++// tile_bytes / n_slots, and the slot pool) are assigned at Rung 2 alongside the ++// double-buffer kernel that consumes them. This keeps the table compute-inert: ++// nothing on the attention path reads layer_tactic until Rung 2, so the graph ++// at small ctx stays byte-identical to Rung 0 / vanilla. ++void llama_kv_cache::build_rolling_kv_plan(float eff_bw_gbps) { ++ layer_tactic.assign(layers.size(), headinfer_tactic::GPU_RESIDENT); ++ rkv_plan = rolling_kv_plan{}; ++ rkv_plan.eff_bw_gbps = eff_bw_gbps; // Stage 3d-S2 (#352): MEASURED host->device PCIe bandwidth (0 = unknown) ++ ++ // opencoti F5 M7 Rolling KV — M7-E E1 (#305): stream-by-default. An ++ // over-budget layer (gpu_heads > 0 ⇒ KV did not fit resident) is served by ++ // the GPU_STREAM tactic — the custom streaming-FA op that tiles attention ++ // over the key dimension with a cross-tile online-softmax — instead of ++ // CPU_SPILL (the W2 iqk CPU-FA engine). ++ // ++ // The S-ladder (S1→S3d) proved GPU_STREAM byte-identical on f16 and coherent ++ // on q8_0 (dequant-on-lift) incl. the n_head_cpu>1 Gemma split, so GPU_STREAM ++ // is the DEFAULT relief tactic (Decision-4 graduated relief). CPU_SPILL (the W2 ++ // iqk CPU-FA path) remains in-tree as a tactic but is no longer auto-selected. ++ // The decision is made ONCE at construction here; the runtime promote/demote- ++ // under-load scheduler is the deferred E2. ++ const headinfer_tactic spill_tactic = headinfer_tactic::GPU_STREAM; ++ ++ // opencoti F5 M7 Stage 3c-5 — POSITION_WINDOW takes precedence: a window layer ++ // (window_cells > 0) keeps the bulk device-resident [0,wc) and streams only the ++ // overflow tail [wc,kv) via the two-region streaming op. It is mutually exclusive ++ // with the M2/E1 head split (gpu_heads == 0 whenever window_cells > 0). ++ for (size_t j = 0; j < layers.size(); ++j) { ++ const headinfer_tactic t = ++ layers[j].window_cells > 0 ? headinfer_tactic::POSITION_WINDOW : ++ layers[j].gpu_heads > 0 ? spill_tactic : ++ headinfer_tactic::GPU_RESIDENT; ++ layer_tactic[j] = t; ++ rkv_plan.hist[(int) t]++; ++ } ++ ++ // "active" tracks whether any *streaming* tactic is engaged. ++ rkv_plan.active = ++ rkv_plan.hist[(int) headinfer_tactic::GPU_STREAM] > 0 || ++ rkv_plan.hist[(int) headinfer_tactic::GPU_STREAM_SUBTILE] > 0 || ++ rkv_plan.hist[(int) headinfer_tactic::POSITION_WINDOW] > 0; ++ ++ LLAMA_LOG_INFO("%s: rolling-kv tactic table: %zu KV layers — " ++ "GPU_RESIDENT=%u GPU_STREAM=%u SUBTILE=%u CPU_SPILL=%u POSITION_WINDOW=%u CPU_FA_TAIL=%u " ++ "(relief=%s; M7-E E1 / Stage 3c-5)\n", ++ __func__, layers.size(), ++ rkv_plan.hist[0], rkv_plan.hist[1], rkv_plan.hist[2], rkv_plan.hist[3], rkv_plan.hist[4], rkv_plan.hist[5], ++ spill_tactic == headinfer_tactic::GPU_STREAM ? "GPU_STREAM" : "CPU_SPILL"); ++ ++ // opencoti F5 M7 Stage 3d-S1 (#351) — run the compute microbench ONLY when a ++ // POSITION_WINDOW layer exists (overflow is actually happening). At small ctx where ++ // every layer is GPU_RESIDENT the boot stays byte-identical (no probe, no extra log). ++ // The measured fa_ms_per_cell is consumed by the Stage-3d-S2 selector; here we just ++ // populate + log it (Rung-1 compute-inert: the attention path still reads only the ++ // tactic table, unchanged). ++ if (rkv_plan.hist[(int) headinfer_tactic::POSITION_WINDOW] > 0) { ++ int32_t bench_il = -1; ++ uint32_t bench_wc = 0; ++ ggml_type tk = GGML_TYPE_F16; ++ ggml_type tv = GGML_TYPE_F16; ++ for (size_t j = 0; j < layers.size(); ++j) { ++ if (layers[j].window_cells > 0 && !layers[j].k_per_stream.empty() && !layers[j].v_per_stream.empty()) { ++ bench_il = (int32_t) layers[j].il; ++ bench_wc = layers[j].window_cells; ++ tk = layers[j].k_per_stream[0]->type; ++ tv = layers[j].v_per_stream[0]->type; ++ break; ++ } ++ } ++ if (bench_il >= 0) { ++ const int64_t bench_cells = 8192; // 256-aligned, ~32 MiB transient K+V; amortizes launch slack ++ const float probe_ms = opencoti_fa_compute_probe_ms(model, hparams, bench_il, tk, tv, bench_cells, /*on_cpu=*/false); ++ if (probe_ms > 0.0f) { ++ rkv_plan.fa_ms_per_cell = probe_ms / (float) bench_cells; ++ rkv_plan.compute_ms = rkv_plan.fa_ms_per_cell * (float) kv_size_max; // per-layer FA ms @ full ctx (reference) ++ LLAMA_LOG_INFO("%s: compute microbench — FA %.3f ms / %lld cells = %.6f ms/cell " ++ "(global layer %d, %s K / %s V); est %.2f ms/layer @ %u cells\n", __func__, ++ (double) probe_ms, (long long) bench_cells, (double) rkv_plan.fa_ms_per_cell, ++ bench_il, ggml_type_name(tk), ggml_type_name(tv), ++ (double) rkv_plan.compute_ms, kv_size_max); ++ } else { ++ LLAMA_LOG_INFO("%s: compute microbench — unavailable (no GPU probe); Stage-3d-S2 selector " ++ "will use the analytic/stream default\n", __func__); ++ } ++ ++ // opencoti F5 M7 Stage 3d-S2 (#352) — overflow tail-tactic crossover selector. ++ // Runs only when the GPU microbench succeeded (fa_ms_per_cell measured). Decides, from ++ // MEASURED hardware (GPU fa_ms_per_cell, CPU cpu_fa_ms_per_cell, eff_bw_gbps), whether ++ // the host overflow tail is best STREAMED to the device (POSITION_WINDOW — hidden under ++ // FA compute on fast links) or FLASH-ATTENDED on the CPU (CPU_FA_TAIL — no DMA, wins on ++ // slow links). Nothing hardcoded — the only constant is the documented 0.8 overlap ++ // safety (pcie-profile.h:8). INFORMATIONAL at S2: layer_tactic stays POSITION_WINDOW so ++ // the forward is byte-unchanged; rkv_plan.tail_tactic carries the choice for the S2 gate ++ // and the S3 CPU-FA-tail execution branch. ++ if (rkv_plan.fa_ms_per_cell > 0.0f) { ++ // CPU companion probe — same synthetic FA on the host CPU backend (ggml-cpu / iqk). ++ const float cpu_probe_ms = opencoti_fa_compute_probe_ms(model, hparams, bench_il, tk, tv, bench_cells, /*on_cpu=*/true); ++ if (cpu_probe_ms > 0.0f) { ++ rkv_plan.cpu_fa_ms_per_cell = cpu_probe_ms / (float) bench_cells; ++ } ++ ++ const uint32_t wc = bench_wc; ++ const uint32_t n_tail = (kv_size_max > wc) ? (kv_size_max - wc) : 0u; ++ const size_t per_cell_bytes = ++ (size_t) ggml_row_size(tk, hparams.n_embd_k_gqa(bench_il)) + ++ (size_t) ggml_row_size(tv, hparams.n_embd_v_gqa(bench_il)); ++ const double tail_bytes = (double) n_tail * (double) per_cell_bytes; ++ const double fa = (double) rkv_plan.fa_ms_per_cell; ++ const double cfa = (double) rkv_plan.cpu_fa_ms_per_cell; ++ ++ // Per-layer, per-decode-step models (GB/s = 1e6 bytes/ms): ++ // stream : the GPU must FA over ALL kv_size keys anyway; the tail DMA overlaps ++ // under that compute → stream_ms = max(fa*kv_size, tail_transfer_ms) ++ // cpu-tail: GPU does only the window FA; the CPU does the tail FA concurrently, ++ // no DMA → cpu_fa_tail_ms = max(fa*wc, cfa*n_tail) ++ const double resident_compute_ms = fa * (double) kv_size_max; ++ const double tail_transfer_ms = (eff_bw_gbps > 0.0f) ++ ? tail_bytes / ((double) eff_bw_gbps * 1.0e6) : 0.0; ++ // tile_bytes_max = bytes hideable under the FA compute (compute_ms*eff_bw*0.8). ++ rkv_plan.tile_bytes = (eff_bw_gbps > 0.0f) ++ ? (size_t) (resident_compute_ms * (double) eff_bw_gbps * 1.0e6 * 0.8) : 0; ++ const double stream_ms = std::max(resident_compute_ms, tail_transfer_ms); ++ ++ headinfer_tactic chosen = headinfer_tactic::POSITION_WINDOW; ++ const char * why = "stream (default)"; ++ double cpu_fa_tail_ms = 0.0; ++ if (eff_bw_gbps <= 0.0f) { ++ why = "stream (bandwidth unknown)"; ++ } else if (tail_bytes <= (double) rkv_plan.tile_bytes) { ++ why = "stream (tail <= tile_bytes_max -> hidden under FA compute)"; ++ } else if (cfa > 0.0f) { ++ // opencoti #354 — CPU_FA_TAIL is GATED OUT of auto-select. We still MODEL its ++ // cost (for the audit-trail log + the physics crossover record) but never CHOOSE ++ // it: routing the host tail through CPU flash-attention fails the single-needle ++ // RULER retrieval bar. The scalar _f16_one_chunk ceilings at niah~80 (q8=60), and ++ // the iqk-wired tail partial measured niah=0.0 — while the GPU-stream ++ // POSITION_WINDOW path holds niah=100. On a slow link this keeps streaming the ++ // tail (some decode tps for correctness — the right trade). Re-enable only when a ++ // CPU tail FA is proven >= vanilla-2pp on niah (the #354 gate). ++ cpu_fa_tail_ms = std::max(fa * (double) wc, cfa * (double) n_tail); ++ // CPU_FA_TAIL stays MODELED here (audit log) but is gated OUT of auto-select ++ // (#354): the doctrine-clean GPU-stream POSITION_WINDOW path holds niah=100, so ++ // even on a slow link we keep streaming the tail — trading some decode tps for ++ // correctness. The tactic + op remain in-tree, dormant, for a future CPU tail FA ++ // that is proven >= vanilla-2pp on niah (the #354 re-enable gate). ++ why = (cpu_fa_tail_ms < stream_ms) ++ ? "stream (CPU_FA_TAIL gated out #354 — faster but loses the needle)" ++ : "stream (<= cpu-fa tail)"; ++ } else { ++ why = "stream (no CPU probe -> CPU_FA_TAIL unmodeled)"; ++ } ++ rkv_plan.tail_tactic = chosen; ++ ++ LLAMA_LOG_INFO("%s: Stage-3d-S2 selector — eff_bw=%.2f GB/s; window=%u tail=%u cells (%.1f MiB); " ++ "fa=%.6f cpu_fa=%.6f ms/cell; resident_compute=%.3f tail_transfer=%.3f stream=%.3f cpu_fa_tail=%.3f ms; " ++ "tile_bytes_max=%.1f MiB -> tail_tactic=%s [%s]\n", __func__, ++ (double) eff_bw_gbps, wc, n_tail, tail_bytes / (1024.0*1024.0), ++ fa, cfa, resident_compute_ms, tail_transfer_ms, stream_ms, cpu_fa_tail_ms, ++ (double) rkv_plan.tile_bytes / (1024.0*1024.0), ++ chosen == headinfer_tactic::CPU_FA_TAIL ? "CPU_FA_TAIL" : "POSITION_WINDOW", why); ++ } ++ } ++ } ++} ++ ++// opencoti F5 M7 Rolling KV — Rung 1. Tactic for model layer `il`. Empty table ++// (Rung 0 path / no auto decision) ⇒ GPU_RESIDENT. ++// opencoti F5 M7 Stage 3d-S3 (#353): the S2 selector stored its overflow tail-tactic ++// choice (stream vs CPU-FA) in rkv_plan.tail_tactic (plan-wide; the global windowed ++// layers are homogeneous). The graph reads this to route the CPU-FA-tail branch. ++bool llama_kv_cache::headinfer_tail_is_cpu_fa() const { ++ return rkv_plan.tail_tactic == headinfer_tactic::CPU_FA_TAIL; ++} ++ ++llama_kv_cache::headinfer_tactic llama_kv_cache::get_layer_tactic(int32_t il) const { ++ if (layer_tactic.empty()) { ++ return headinfer_tactic::GPU_RESIDENT; ++ } ++ const auto it = map_layer_ids.find(il); ++ if (it == map_layer_ids.end() || (size_t) it->second >= layer_tactic.size()) { ++ return headinfer_tactic::GPU_RESIDENT; ++ } ++ return layer_tactic[it->second]; ++} ++ + bool llama_kv_cache::headinfer_split_active(int32_t il) const { + const auto it = map_layer_ids.find(il); + if (it == map_layer_ids.end()) { +@@ -2184,6 +2878,159 @@ ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t + return result; + } + ++// opencoti F5 M7 Stage 3c-5 — POSITION_WINDOW perf-path accessors. Unlike the ++// get_k/get_v dim-2 concat (correctness fallback), these return the device window ++// and pinned-host tail regions SEPARATELY so build_attn_mha_position_window can run ++// a window FA over the resident bulk and stream only the overflow tail. Layout is ++// identical to get_k_gpu ([n_embd_head_k, n_head_kv, cells, n_stream]); the caller ++// applies the same permute(0,2,1,3) the FA path uses. The device window tensor has ++// ne[1] == window_cells, so its dim-3 (stream/position) stride is computed off ++// k_s->ne[1] (the audit point — NOT kv_size); the host tail likewise off kc_s->ne[1]. ++ggml_tensor * llama_kv_cache::get_k_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.window_cells > 0 && "get_k_window called without an active position window"); ++ ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_k_gqa2 = (uint64_t) n_head_kv * n_embd_head_k; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_win = n_kv < wc ? n_kv : wc; ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * k_s = layer.k_per_stream[s_cache]; ++ return ggml_view_4d(ctx, k_s, ++ n_embd_head_k, n_head_kv, n_win, 1, ++ ggml_row_size(k_s->type, n_embd_head_k), ++ ggml_row_size(k_s->type, n_embd_k_gqa2), ++ ggml_row_size(k_s->type, n_embd_k_gqa2 * (uint64_t) k_s->ne[1]), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++} ++ ++ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.window_cells > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0] ++ && "get_k_tail called without an active position window"); ++ ++ const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_k_gqa2 = (uint64_t) n_head_kv * n_embd_head_k; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_tail = n_kv > wc ? n_kv - wc : 0; ++ if (n_tail == 0) { ++ return nullptr; // no overflow → fully resident window, single-region FA ++ } ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ return ggml_view_4d(ctx, kc_s, ++ n_embd_head_k, n_head_kv, n_tail, 1, ++ ggml_row_size(kc_s->type, n_embd_head_k), ++ ggml_row_size(kc_s->type, n_embd_k_gqa2), ++ ggml_row_size(kc_s->type, n_embd_k_gqa2 * (uint64_t) kc_s->ne[1]), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++} ++ ++ggml_tensor * llama_kv_cache::get_v_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.window_cells > 0 && "get_v_window called without an active position window"); ++ GGML_ASSERT(!v_trans && "position window requires non-transposed V"); ++ ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_v_gqa2 = (uint64_t) n_head_kv * n_embd_head_v; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_win = n_kv < wc ? n_kv : wc; ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * v_s = layer.v_per_stream[s_cache]; ++ return ggml_view_4d(ctx, v_s, ++ n_embd_head_v, n_head_kv, n_win, 1, ++ ggml_row_size(v_s->type, n_embd_head_v), ++ ggml_row_size(v_s->type, n_embd_v_gqa2), ++ ggml_row_size(v_s->type, n_embd_v_gqa2 * (uint64_t) v_s->ne[1]), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++} ++ ++ggml_tensor * llama_kv_cache::get_v_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ GGML_ASSERT(layer.window_cells > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[0] ++ && "get_v_tail called without an active position window"); ++ GGML_ASSERT(!v_trans && "position window requires non-transposed V"); ++ ++ const uint32_t n_embd_head_v = hparams.n_embd_head_v(il); ++ const uint32_t n_head_kv = hparams.n_head_kv(il); ++ const uint64_t n_embd_v_gqa2 = (uint64_t) n_head_kv * n_embd_head_v; ++ const uint32_t wc = layer.window_cells; ++ const uint32_t n_tail = n_kv > wc ? n_kv - wc : 0; ++ if (n_tail == 0) { ++ return nullptr; ++ } ++ ++ auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[s_cache]; ++ return ggml_view_4d(ctx, vc_s, ++ n_embd_head_v, n_head_kv, n_tail, 1, ++ ggml_row_size(vc_s->type, n_embd_head_v), ++ ggml_row_size(vc_s->type, n_embd_v_gqa2), ++ ggml_row_size(vc_s->type, n_embd_v_gqa2 * (uint64_t) vc_s->ne[1]), ++ 0); ++ }; ++ ++ ggml_tensor * result = build_stream_view(sinfo.s0); ++ for (uint32_t s = sinfo.s0 + 1; s <= sinfo.s1; ++s) { ++ result = ggml_concat(ctx, result, build_stream_view(s), 3); ++ } ++ return result; ++} ++ ++bool llama_kv_cache::headinfer_window_active(int32_t il) const { ++ const auto it = map_layer_ids.find(il); ++ if (it == map_layer_ids.end()) { ++ return false; ++ } ++ const auto & layer = layers[it->second]; ++ return layer.window_cells > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0] != nullptr; ++} ++ ++uint32_t llama_kv_cache::headinfer_window_cells(int32_t il) const { ++ const auto it = map_layer_ids.find(il); ++ if (it == map_layer_ids.end()) { ++ return 0; ++ } ++ return layers[it->second].window_cells; ++} ++ + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + +@@ -2258,6 +3105,65 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + return tie; + } + ++ // opencoti F5 M7 Stage 3c — position-window scatter. Route each row to the ++ // device window (dest cell idx < wc) or the pinned host tail (idx >= wc, at ++ // local index idx-wc). set_input_k_idxs has already written window-local ++ // (idx) / tail-local (idx-wc) values into k_idxs. For the append-only base ++ // cache the destination cells are position-sorted within a stream, so the ++ // window rows form a contiguous prefix [0,kw) and the tail rows the suffix ++ // [kw,tps). Two set_rows (window + tail) per stream, tied into one dep node. ++ // ++ // The split point kw is read from sinfo.idxs at graph-build. The full-cache ++ // reserve context installs a DUMMY sinfo whose idxs[s] has a single element ++ // while the worst-case ubatch has many tokens — so when idxs[s].size() != ++ // tokens_per_stream we are at reserve and build a conservative both-full ++ // graph for memory planning (no kernel runs at reserve, so index values are ++ // irrelevant; only the topology must cover the real graph, which uses <= the ++ // same two set_rows per stream). ++ if (layer.window_cells > 0 && !layer.k_cpu_per_stream.empty() && layer.k_cpu_per_stream[0]) { ++ const uint32_t wc = layer.window_cells; ++ ggml_tensor * k_cur2 = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0); ++ const uint32_t batch_ns = sinfo.n_stream(); ++ GGML_ASSERT(batch_ns >= 1); ++ const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ auto accumulate = [&](ggml_tensor * store) { ++ if (!tie) { tie = store; return; } ++ tie = ggml_add(ctx, ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ }; ++ auto emit = [&](ggml_tensor * dst, int64_t row0, int64_t nrows, size_t base) { ++ if (nrows <= 0) return; ++ ggml_tensor * cur = ggml_view_2d(ctx, k_cur2, n_embd_gqa, nrows, ++ k_cur2->nb[1], (base + (size_t) row0) * k_cur2->nb[1]); ++ ggml_tensor * idx = ggml_view_1d(ctx, k_idxs, nrows, (base + (size_t) row0) * k_idxs->nb[0]); ++ accumulate(ggml_set_rows(ctx, dst, cur, idx)); ++ }; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[s]]; ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[sinfo.strm[s]]; ++ const size_t base = (size_t) s * tokens_per_stream; ++ if (sinfo.idxs[s].size() != (size_t) tokens_per_stream) { ++ // reserve dummy sinfo → conservative both-full graph for planning ++ emit(k_s, 0, tokens_per_stream, base); ++ emit(kc_s, 0, tokens_per_stream, base); ++ continue; ++ } ++ int64_t kw = 0; ++ while (kw < tokens_per_stream && (uint32_t) sinfo.idxs[s][kw] < wc) ++kw; ++ for (int64_t r = kw; r < tokens_per_stream; ++r) { ++ GGML_ASSERT((uint32_t) sinfo.idxs[s][r] >= wc ++ && "position-window scatter requires position-sorted destination idxs"); ++ } ++ emit(k_s, 0, kw, base); // window prefix [0,kw) ++ emit(kc_s, kw, tokens_per_stream - kw, base); // tail suffix [kw,tps) ++ } ++ return tie; ++ } ++ + k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0); + + // opencoti F4 M3 Phase 4-C: per-stream scatter via N separate +@@ -2389,6 +3295,46 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + GGML_ASSERT(batch_ns >= 1); + + if (!v_trans) { ++ // opencoti F5 M7 Stage 3c — position-window scatter (mirror of cpy_k). ++ // window mode is gated to !v_trans, so V is always reached here. Same ++ // window prefix / tail suffix split + reserve-dummy guard as cpy_k. ++ if (layer.window_cells > 0 && !layer.v_cpu_per_stream.empty() && layer.v_cpu_per_stream[0]) { ++ const uint32_t wc = layer.window_cells; ++ ggml_tensor * v_cur2 = ggml_view_2d(ctx, v_cur, n_embd_gqa, n_tokens, v_cur->nb[2], 0); ++ const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); ++ GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); ++ ++ ggml_tensor * tie = nullptr; ++ auto accumulate = [&](ggml_tensor * store) { ++ if (!tie) { tie = store; return; } ++ tie = ggml_add(ctx, ++ ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ }; ++ auto emit = [&](ggml_tensor * dst, int64_t row0, int64_t nrows, size_t base) { ++ if (nrows <= 0) return; ++ ggml_tensor * cur = ggml_view_2d(ctx, v_cur2, n_embd_gqa, nrows, ++ v_cur2->nb[1], (base + (size_t) row0) * v_cur2->nb[1]); ++ ggml_tensor * idx = ggml_view_1d(ctx, v_idxs, nrows, (base + (size_t) row0) * v_idxs->nb[0]); ++ accumulate(ggml_set_rows(ctx, dst, cur, idx)); ++ }; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[s]]; ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[sinfo.strm[s]]; ++ const size_t base = (size_t) s * tokens_per_stream; ++ if (sinfo.idxs[s].size() != (size_t) tokens_per_stream) { ++ emit(v_s, 0, tokens_per_stream, base); ++ emit(vc_s, 0, tokens_per_stream, base); ++ continue; ++ } ++ int64_t kw = 0; ++ while (kw < tokens_per_stream && (uint32_t) sinfo.idxs[s][kw] < wc) ++kw; ++ emit(v_s, 0, kw, base); ++ emit(vc_s, kw, tokens_per_stream - kw, base); ++ } ++ return tie; ++ } ++ + v_cur = ggml_view_2d(ctx, v_cur, n_embd_gqa, n_tokens, v_cur->nb[2], 0); + + if (batch_ns == 1) { +@@ -2535,9 +3481,15 @@ void llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ub + // global indices wrote out-of-bounds against the per-stream + // tensor and crashed CUDA decode. + ++ // opencoti F5 M7 Stage 3c — position-window local indexing. When the cache is ++ // windowed, a destination cell idx >= wc lives in the host tail at local index ++ // idx-wc; idx < wc stays window-local. wc is uniform across the cache's layers. ++ // wc == 0 (no window) ⇒ byte-identical to the vanilla local indexing below. ++ const uint32_t wc_k = layers.empty() ? 0u : layers.front().window_cells; + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + for (uint32_t i = 0; i < sinfo.size(); ++i) { +- data[s*sinfo.size() + i] = sinfo.idxs[s][i]; ++ const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; ++ data[s*sinfo.size() + i] = (wc_k > 0 && idx >= wc_k) ? (int64_t) (idx - wc_k) : (int64_t) idx; + } + } + } +@@ -2555,9 +3507,14 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub + // byte-identical regardless (sinfo.strm[0] == 0 makes offs == 0). + + if (!v_trans) { ++ // opencoti F5 M7 Stage 3c — position-window local indexing (mirror of ++ // set_input_k_idxs). window mode is gated to !v_trans, so the tail-local ++ // adjustment only applies here. wc == 0 ⇒ byte-identical to vanilla. ++ const uint32_t wc_v = layers.empty() ? 0u : layers.front().window_cells; + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + for (uint32_t i = 0; i < sinfo.size(); ++i) { +- data[s*sinfo.size() + i] = sinfo.idxs[s][i]; ++ const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; ++ data[s*sinfo.size() + i] = (wc_v > 0 && idx >= wc_v) ? (int64_t) (idx - wc_v) : (int64_t) idx; + } + } + } else { +@@ -3234,6 +4191,34 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t + io.write_tensor(k, range.first * k_row_gpu, range_size * k_row_gpu); + io.write_tensor(k_cpu, range.first * k_row_cpu, range_size * k_row_cpu); + } ++ } else if (layer.window_cells > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[cr.strm]) { ++ // opencoti F5 M7 Stage 3c-6 — position-window split-aware state write (K). ++ // In window mode k (k_stream) holds the resident cells [0,wc) and k_cpu the ++ // overflow tail [wc,kv_size) at local idx-wc; both carry ALL heads, so each ++ // cell's row is k_size_row. Emit each range's cells in LOGICAL position order ++ // (window prefix [a,min(b,wc)) then tail suffix [max(a,wc),b)), so the on-disk ++ // bytes are byte-identical to a vanilla / M2 save → a window save loads into a ++ // resident cache and vice-versa (cross-residency-mode compatible). ++ auto * k_cpu = layer.k_cpu_per_stream[cr.strm]; ++ const uint32_t wc = layer.window_cells; ++ GGML_ASSERT((uint64_t) ggml_row_size(k_cpu->type, n_embd_k_gqa) == k_size_row); ++ for (const auto & range : cr.data) { ++ const uint32_t a = range.first, b = range.second; ++ const uint32_t w_hi = b < wc ? b : wc; // window cells [a, w_hi) ++ const uint32_t t_lo = a > wc ? a : wc; // tail cells [t_lo, b) ++ size_t wrote = 0; ++ if (w_hi > a) { ++ io.write_tensor(k, (size_t) a * k_size_row, (size_t) (w_hi - a) * k_size_row); ++ wrote += (size_t) (w_hi - a) * k_size_row; ++ } ++ if (b > t_lo) { ++ io.write_tensor(k_cpu, (size_t) (t_lo - wc) * k_size_row, (size_t) (b - t_lo) * k_size_row); ++ wrote += (size_t) (b - t_lo) * k_size_row; ++ } ++ GGML_ASSERT(wrote == (size_t) (b - a) * k_size_row && "3c-6 K window/tail parity"); ++ } + } else { + // Read each range of cells of k_size length and write out + for (const auto & range : cr.data) { +@@ -3283,6 +4268,30 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t + io.write_tensor(v, range.first * v_row_gpu, range_size * v_row_gpu); + io.write_tensor(v_cpu, range.first * v_row_cpu, range_size * v_row_cpu); + } ++ } else if (layer.window_cells > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[cr.strm]) { ++ // opencoti F5 M7 Stage 3c-6 — position-window split-aware state write (V). ++ // Mirror of the K branch: window prefix from v (k_stream) then tail suffix ++ // from v_cpu, in logical cell order → byte-identical on-disk layout. ++ auto * v_cpu = layer.v_cpu_per_stream[cr.strm]; ++ const uint32_t wc = layer.window_cells; ++ GGML_ASSERT((uint64_t) ggml_row_size(v_cpu->type, n_embd_v_gqa) == v_size_row); ++ for (const auto & range : cr.data) { ++ const uint32_t a = range.first, b = range.second; ++ const uint32_t w_hi = b < wc ? b : wc; ++ const uint32_t t_lo = a > wc ? a : wc; ++ size_t wrote = 0; ++ if (w_hi > a) { ++ io.write_tensor(v, (size_t) a * v_size_row, (size_t) (w_hi - a) * v_size_row); ++ wrote += (size_t) (w_hi - a) * v_size_row; ++ } ++ if (b > t_lo) { ++ io.write_tensor(v_cpu, (size_t) (t_lo - wc) * v_size_row, (size_t) (b - t_lo) * v_size_row); ++ wrote += (size_t) (b - t_lo) * v_size_row; ++ } ++ GGML_ASSERT(wrote == (size_t) (b - a) * v_size_row && "3c-6 V window/tail parity"); ++ } + } else { + // Read each range of cells of v_size length and write out + for (const auto & range : cr.data) { +@@ -3530,6 +4539,28 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32 + io.read_tensor(k_cpu, sinfo.idxs[0][i] * k_row_cpu, k_row_cpu); + } + } ++ } else if (layer.window_cells > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[strm]) { ++ // opencoti F5 M7 Stage 3c-6 — symmetric position-window read (K). The ++ // on-disk stream is cell_count rows in logical position order (vanilla ++ // layout, written above); route each row to the device window (dest ++ // position < wc) or the host tail (>= wc, local pos-wc) by its ++ // destination cell position. Handles contiguous and scattered sinfo ++ // uniformly (a contiguous run can still straddle wc). New io.read_tensor ++ // idiom (no ptr-returning io.read in this base): each read_tensor ++ // consumes one row from the stream in logical order. ++ auto * k_cpu = layer.k_cpu_per_stream[strm]; ++ const uint32_t wc = layer.window_cells; ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ const uint32_t p = sinfo.is_contiguous() ++ ? (sinfo.head() + i) : (uint32_t) sinfo.idxs[0][i]; ++ if (p < wc) { ++ io.read_tensor(k, (size_t) p * k_size_row, k_size_row); ++ } else { ++ io.read_tensor(k_cpu, (size_t) (p - wc) * k_size_row, k_size_row); ++ } ++ } + } else if (sinfo.is_contiguous()) { + // Fast path: contiguous cells, single memcpy + io.read_tensor(k, sinfo.head() * k_size_row, cell_count * k_size_row); +@@ -3599,6 +4630,24 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32 + io.read_tensor(v_cpu, sinfo.idxs[0][i] * v_row_cpu, v_row_cpu); + } + } ++ } else if (layer.window_cells > 0 ++ && !layer.v_cpu_per_stream.empty() ++ && layer.v_cpu_per_stream[strm]) { ++ // opencoti F5 M7 Stage 3c-6 — symmetric position-window read (V). ++ // Mirror of the K branch: route each row to v (window) or v_cpu ++ // (tail, local pos-wc) by its destination cell position. New ++ // io.read_tensor idiom — one row per call, sequential stream order. ++ auto * v_cpu = layer.v_cpu_per_stream[strm]; ++ const uint32_t wc = layer.window_cells; ++ for (uint32_t i = 0; i < cell_count; ++i) { ++ const uint32_t p = sinfo.is_contiguous() ++ ? (sinfo.head() + i) : (uint32_t) sinfo.idxs[0][i]; ++ if (p < wc) { ++ io.read_tensor(v, (size_t) p * v_size_row, v_size_row); ++ } else { ++ io.read_tensor(v_cpu, (size_t) (p - wc) * v_size_row, v_size_row); ++ } ++ } + } else if (sinfo.is_contiguous()) { + // Fast path: contiguous cells, single memcpy + io.read_tensor(v, sinfo.head() * v_size_row, cell_count * v_size_row); +@@ -3787,6 +4836,27 @@ ggml_tensor * llama_kv_cache_context::get_v_cpu(ggml_context * ctx, int32_t il) + return kv->get_v_cpu(ctx, il, n_kv, sinfos[i_cur]); + } + ++// opencoti F5 M7 Stage 3c-5 — per-batch wrappers around the position-window accessors. ++ggml_tensor * llama_kv_cache_context::get_k_window(ggml_context * ctx, int32_t il) const { ++ return kv->get_k_window(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_v_window(ggml_context * ctx, int32_t il) const { ++ return kv->get_v_window(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_k_tail(ggml_context * ctx, int32_t il) const { ++ return kv->get_k_tail(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++ggml_tensor * llama_kv_cache_context::get_v_tail(ggml_context * ctx, int32_t il) const { ++ return kv->get_v_tail(ctx, il, n_kv, sinfos[i_cur]); ++} ++ ++bool llama_kv_cache_context::headinfer_window_active(int32_t il) const { ++ return kv->headinfer_window_active(il); ++} ++ + bool llama_kv_cache_context::headinfer_split_active(int32_t il) const { + return kv->headinfer_split_active(il); + } +@@ -3795,6 +4865,16 @@ uint32_t llama_kv_cache_context::headinfer_gpu_heads(int32_t il) const { + return kv->headinfer_gpu_heads(il); + } + ++// opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304). ++llama_kv_cache::headinfer_tactic llama_kv_cache_context::get_layer_tactic(int32_t il) const { ++ return kv->get_layer_tactic(il); ++} ++ ++// opencoti F5 M7 Stage 3d-S3 (#353). ++bool llama_kv_cache_context::headinfer_tail_is_cpu_fa() const { ++ return kv->headinfer_tail_is_cpu_fa(); ++} ++ + ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { + return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]); + } +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -116,6 +116,21 @@ public: + // to host, streamed back per step). 1.0 = off. Effective + // only for offloaded layers with !v_trans (flash-attn). + float headinfer_gpu_heads_frac, ++ // opencoti F5 M7 Rolling KV — Rung 0 — see docs/features/rolling_kv.md ++ // VRAM budget cap (MiB) for auto residency. 0 = use all free ++ // VRAM (maximize). Only consulted when headinfer_gpu_heads_frac ++ // is negative (auto); ignored for an explicit fraction. ++ uint32_t vram_target_mib, ++ // opencoti F5 M7 Rolling KV — Rung 2 (Stage 3d-S2 #352). Effective ++ // host->device PCIe bandwidth (GB/s), resolved at boot from the W1 ++ // pcie_profile (ReBAR probe / sysfs / --pcie-bw-gbps override). Fed to ++ // build_rolling_kv_plan so the tail-tactic selector sizes the stream ++ // vs CPU-FA crossover from MEASURED bandwidth. 0 = unknown (stream default). ++ float pcie_bw_gbps, ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. ++ // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), ++ // 1 = head (force M2 split), 2 = window (force POSITION_WINDOW). ++ uint32_t kv_residency_mode, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -234,6 +249,28 @@ public: + bool headinfer_split_active(int32_t il) const; + uint32_t headinfer_gpu_heads (int32_t il) const; + ++ // opencoti F5 M7 Stage 3c-5 — position-window accessors for the POSITION_WINDOW ++ // perf path (build_attn_mha_position_window). Unlike get_k/get_v (which dim-2 ++ // concat the window + tail back into one [n_embd, n_kv] view — the correctness ++ // fallback), these return the RAW resident-window and pinned-host-tail region ++ // views separately so the streaming op can keep the bulk device-resident and DMA ++ // only the overflow. ALL heads, sliced on the POSITION axis: ++ // get_k/v_window → device view, n_win = min(n_kv, window_cells) cells [0,n_win) ++ // get_k/v_tail → host view, n_tail = n_kv > wc ? n_kv - wc : 0 cells, or ++ // nullptr when n_kv <= window_cells (no overflow → resident). ++ // Callers MUST gate on headinfer_window_active(il); for non-window layers the ++ // window accessors fall back to the equivalent of get_k/get_v and tail is nullptr. ++ ggml_tensor * get_k_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_v_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_k_tail (ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ ggml_tensor * get_v_tail (ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ bool headinfer_window_active(int32_t il) const; ++ uint32_t headinfer_window_cells (int32_t il) const; ++ // opencoti F5 M7 Stage 3d-S3 (#353): true when the S2 selector chose CPU_FA_TAIL ++ // (FA the host tail on the CPU, no DMA) for the windowed layers — read by the ++ // graph to route build_attn_mha_position_window down the CPU-partial branch. ++ bool headinfer_tail_is_cpu_fa() const; ++ + // store k_cur and v_cur in the cache based on the provided head location + ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const; + ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const; +@@ -277,6 +314,46 @@ public: + void set_input_k_rot(ggml_tensor * dst) const; + void set_input_v_rot(ggml_tensor * dst) const; + ++ // opencoti F5 M7 Rolling KV — Rung 1 (M7-B/C) runtime tactic table. ++ // Per-KV-layer residency tactic, decided from VRAM budget + PCIe bandwidth ++ // at construction and re-decidable at runtime by later rungs. Rung 1 ++ // POPULATES + LOGS this table and the streaming plan but does NOT consume ++ // them in the compute path (the build_attn_mha dispatch + slot pool land at ++ // Rung 2 / M7-D/E). At small ctx where KV fits, every layer is ++ // GPU_RESIDENT and the graph is byte-identical to Rung 0 / vanilla. ++ enum class headinfer_tactic : uint8_t { ++ GPU_RESIDENT = 0, // full KV on device, no DMA (vanilla fast path) ++ GPU_STREAM = 1, // ping-pong tile streamed per step (Rung 2) ++ GPU_STREAM_SUBTILE = 2, // tile > slot: online-softmax subtile (Rung 2) ++ CPU_SPILL = 3, // M2 head-split CPU half (shipped tactic) ++ POSITION_WINDOW = 4, // Stage 3c: device window [0,wc) + host tail [wc,kv) streamed ++ CPU_FA_TAIL = 5, // Stage 3d-S2: device window [0,wc) + CPU-FA host tail (no DMA) ++ }; ++ ++ struct rolling_kv_plan { ++ bool active = false; // any layer not GPU_RESIDENT-or-spill-only ++ float eff_bw_gbps = 0.0f; // host->device bandwidth (W1 pcie_profile) ++ float compute_ms = 0.0f; // est. FA ms / layer at target ctx ++ float fa_ms_per_cell = 0.0f; // opencoti Stage 3d-S1 (#351): MEASURED GPU flash-attn ms per KV cell (compute microbench). 0 ⇒ unmeasured (selector falls back to the stream default). ++ float cpu_fa_ms_per_cell = 0.0f; // opencoti Stage 3d-S2 (#352): MEASURED CPU flash-attn ms per KV cell (companion probe). 0 ⇒ unmeasured (CPU_FA_TAIL not modeled → stream). ++ size_t tile_bytes = 0; // tile_bytes_max = compute_ms·eff_bw·0.8 (bytes hideable under FA compute) ++ uint32_t n_slots = 0; // ping-pong slots [2,4]; 0 until Rung 2 allocates ++ // opencoti Stage 3d-S2 (#352): the bandwidth-vs-CPU crossover choice for the ++ // OVERFLOW tail, decided from MEASURED hardware (fa_ms_per_cell, cpu_fa_ms_per_cell, ++ // eff_bw_gbps) — nothing hardcoded (auto-adapts PCIe 3.0 x1 … 5.0 x16). Either ++ // POSITION_WINDOW (stream the tail, hidden under FA compute on fast links) or ++ // CPU_FA_TAIL (FA the host tail on the CPU, no DMA, on slow links). Informational at ++ // S2 (layer_tactic stays POSITION_WINDOW so the forward is unchanged); S3 consumes it. ++ headinfer_tactic tail_tactic = headinfer_tactic::POSITION_WINDOW; ++ uint32_t hist[6] = { 0, 0, 0, 0, 0, 0 }; // tactic histogram, indexed by headinfer_tactic (CPU_FA_TAIL==5) ++ }; ++ ++ const rolling_kv_plan & get_rolling_kv_plan() const { return rkv_plan; } ++ ++ // Tactic for KV-cache layer `il` (model layer id). GPU_RESIDENT when the ++ // table is empty (Rung 0 path / no auto decision). ++ headinfer_tactic get_layer_tactic(int32_t il) const; ++ + private: + const llama_model & model; + const llama_hparams & hparams; +@@ -320,6 +397,15 @@ private: + std::vector v_cpu_per_stream; + uint32_t gpu_heads = 0; + ++ // opencoti F5 M7 Stage 3c — position-window residency (mutually exclusive ++ // with the head split above: gpu_heads == 0 whenever this is set). When ++ // window_cells > 0 the device tensors k_per_stream[s]/v_per_stream[s] hold ++ // ALL heads but only the resident window [0, window_cells) — so ne[1] == ++ // window_cells, NOT kv_size (the audit point) — and k_cpu_per_stream[s]/ ++ // v_cpu_per_stream[s] hold ALL heads for the pinned-host tail ++ // [window_cells, kv_size). 0 ⇒ no position window (vanilla / M2 layout). ++ uint32_t window_cells = 0; ++ + // Convenience accessors for unified-mode (n_stream == 1) call + // sites. Returns nullptr in non-unified mode — that's the + // signal that the caller must use per-stream access instead. +@@ -415,6 +501,21 @@ private: + // model layer id -> KV cache layer id + std::unordered_map map_layer_ids; + ++ // opencoti F5 M7 Rolling KV — Rung 1. Per-KV-cache-layer tactic (same index ++ // space as `layers`), and the derived streaming plan. Populated + logged at ++ // construction by build_rolling_kv_plan(); inert until Rung 2. Empty vector ++ // ⇒ every layer GPU_RESIDENT (Rung 0 path). ++ std::vector layer_tactic; ++ rolling_kv_plan rkv_plan; ++ ++ // opencoti F5 M7 Rolling KV — Rung 1. Derive layer_tactic[] + rkv_plan from ++ // the per-layer split decision already baked into `layers` (gpu_heads > 0 ⇒ ++ // CPU_SPILL, else GPU_RESIDENT), then log the histogram. Called once at the ++ // end of construction. Compute-inert: nothing reads the table until Rung 2. ++ // eff_bw_gbps (Stage 3d-S2 #352) feeds the overflow tail-tactic crossover ++ // selector (stream vs CPU-FA); 0 = unknown ⇒ stream default. ++ void build_rolling_kv_plan(float eff_bw_gbps); ++ + size_t total_size() const; + + size_t size_k_bytes() const; +@@ -509,6 +610,22 @@ public: + bool headinfer_split_active(int32_t il) const; + uint32_t headinfer_gpu_heads (int32_t il) const; + ++ // opencoti F5 M7 Stage 3c-5 — per-batch POSITION_WINDOW accessors. Delegate to ++ // the underlying cache's get_k/v_window / get_k/v_tail with the current n_kv + ++ // slot info. get_k/v_tail return nullptr when n_kv <= window_cells (no overflow). ++ ggml_tensor * get_k_window(ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_v_window(ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_k_tail (ggml_context * ctx, int32_t il) const; ++ ggml_tensor * get_v_tail (ggml_context * ctx, int32_t il) const; ++ bool headinfer_window_active(int32_t il) const; ++ ++ // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304). Per-layer tactic for ++ // the streaming dispatch at llama-graph.cpp build_attn. Delegates to the ++ // underlying cache's tactic table (built at ctor by build_rolling_kv_plan). ++ llama_kv_cache::headinfer_tactic get_layer_tactic(int32_t il) const; ++ // opencoti F5 M7 Stage 3d-S3 (#353): delegate the CPU_FA_TAIL tail-tactic query. ++ bool headinfer_tail_is_cpu_fa() const; ++ + // store k_cur and v_cur in the cache based on the provided head location + // note: the heads in k_cur and v_cur should be laid out contiguously in memory + // - k_cur [n_embd_head_k, n_head_k, n_tokens] +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -50,6 +50,12 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + 0, + // opencoti F5 M2 headinfer — hybrid-iswa does not thread headinfer; 1.0 = off. + 1.0f, ++ // opencoti F5 M7 Rolling KV — hybrid-iswa does not thread auto residency; 0 = unused. ++ 0, ++ // opencoti F5 M7 Stage 3d-S2 — hybrid-iswa does not thread PCIe bandwidth; 0 = unknown (stream default). ++ 0.0f, ++ // opencoti F5 M7 Stage 4 — hybrid-iswa does not thread the residency-mode knob; 0 = auto. ++ 0, + n_seq_max, + n_ubatch, + n_pad, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -49,6 +49,12 @@ llama_memory_hybrid::llama_memory_hybrid( + 0, + // opencoti F5 M2 headinfer — hybrid memory does not thread headinfer; 1.0 = off. + 1.0f, ++ // opencoti F5 M7 Rolling KV — hybrid memory does not thread auto residency; 0 = unused. ++ 0, ++ // opencoti F5 M7 Stage 3d-S2 — hybrid memory does not thread PCIe bandwidth; 0 = unknown (stream default). ++ 0.0f, ++ // opencoti F5 M7 Stage 4 — hybrid memory does not thread the residency-mode knob; 0 = auto. ++ 0, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2079,6 +2079,12 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.slot_shrink_idle_ms, + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac, ++ // opencoti F5 M7 Rolling KV — Rung 0 — see docs/features/rolling_kv.md ++ cparams.vram_target_mib, ++ // opencoti F5 M7 Rolling KV — Rung 2 (Stage 3d-S2 #352) ++ cparams.pcie_bw_gbps, ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob ++ cparams.kv_residency_mode, + cparams.n_seq_max, + cparams.n_ubatch, + 1, +@@ -2101,6 +2107,12 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.slot_shrink_idle_ms, + // opencoti F5 M2 headinfer — see docs/features/advanced_kv.md + cparams.headinfer_gpu_heads_frac, ++ // opencoti F5 M7 Rolling KV — Rung 0 — see docs/features/rolling_kv.md ++ cparams.vram_target_mib, ++ // opencoti F5 M7 Rolling KV — Rung 2 (Stage 3d-S2 #352) ++ cparams.pcie_bw_gbps, ++ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob ++ cparams.kv_residency_mode, + cparams.n_seq_max, + 1, + hparams.n_swa, +diff --git a/llama.cpp/tools/server/server.cpp b/llama.cpp/tools/server/server.cpp +--- a/llama.cpp/tools/server/server.cpp ++++ b/llama.cpp/tools/server/server.cpp +@@ -145,6 +145,11 @@ int server_main(int argc, char ** argv, + LOG_INF("%s: pcie profile: %.1f GB/s effective (x%d @ %.1f GT/s, source=%s) — %s\n", + __func__, (double) pp.effective_bw_gbps, pp.link_width, (double) pp.link_gtps, + pp.source.c_str(), pp.detail.c_str()); ++ // opencoti F5 M7 Stage 3d-S2 (#352): write the RESOLVED bandwidth back into params so it ++ // flows params → cparams.pcie_bw_gbps (common.cpp) → the KV-cache ctor → build_rolling_kv_plan's ++ // overflow tail-tactic selector. Under autodetect params.pcie_bw_gbps starts 0; this captures ++ // the detected/override value. ctx_server.load_model(params) (below) reads the updated params. ++ params.pcie_bw_gbps = pp.effective_bw_gbps; + } + + server_http_context ctx_http; diff --git a/patches/0071-state-io-cosmocc.patch b/patches/0071-state-io-cosmocc.patch new file mode 100644 index 0000000000000000000000000000000000000000..6ea05a9b757ea26e238226b02d2d2d94e7f1dd46 --- /dev/null +++ b/patches/0071-state-io-cosmocc.patch @@ -0,0 +1,88 @@ +From: opencoti +Subject: [PATCH 0071] cosmocc state-IO open fix (bug-269) + +opencoti F5 M7 (#349, bug-269). Orthogonal to Rolling KV (0070) — captured as +its own slot. Fixes ALL llama_state_*_save/load_file in the cosmocc/llamafile +build, which were broken before this: llama.cpp/src/llama-mmap.cpp's COSMOCC +llama_file constructor routed every open() through llamafile_open_gguf(), which +pread()s the GGUF magic at offset 0 (EBADF on a write-only "wb" fd) and +zip-routes any non-GGUF read; and write_raw's fwrite(fp) ran with fp==NULL. +KV-state files are neither GGUF nor zip and are opened for write, so model +load was unaffected but state save/load never worked through the binary (the +M2 bug-222 "state round-trip" was thus never actually exercised end-to-end). + +FIX (cosmocc/host-only; no CUDA, no kernel): + * vendors/sources/llamafile/llamafile/llamafile.{c,h} (Mozilla-Ocho llamafile, + Apache-2.0): add llamafile_open_plain() — a plain fopen-backed handle with + no GGUF magic probe and no zip routing, for non-model files. It delegates + to the existing static llamafile_open_file(). + * llama.cpp/src/llama-mmap.cpp: the COSMOCC llama_file ctor falls back to + llamafile_open_plain() when llamafile_open_gguf() returns NULL (models are + unaffected — they still open via the gguf path) and sets fp = + llamafile_fp(lfile) so write_raw has a valid stream. + +Provenance: opencoti-original. Touches Mozilla-Ocho llamafile.{c,h} +(Apache-2.0) and upstream llama.cpp llama-mmap.cpp. Applies after 0070 +(independent file set — order immaterial); reverse-applies clean. +diff --git a/llama.cpp/src/llama-mmap.cpp b/llama.cpp/src/llama-mmap.cpp +--- a/llama.cpp/src/llama-mmap.cpp ++++ b/llama.cpp/src/llama-mmap.cpp +@@ -186,11 +186,25 @@ struct llama_file::impl { + #else + impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) { + #ifdef COSMOCC +- // [llamafile] Use llamafile_open_gguf for all file opening ++ // [llamafile] Use llamafile_open_gguf for model files (raw .gguf, ++ // foo.zip@weights, /zip/ paths, and .llamafile archives via zip routing). + lfile = llamafile_open_gguf(fname, mode); ++ // opencoti bug-269: llamafile_open_gguf is model-oriented — it pread()s the ++ // GGUF magic at offset 0 and routes non-GGUF files into a zip parser. KV ++ // state files (llama_state_*_save_file) are neither GGUF nor zip and are ++ // opened for write ("wb" → pread EBADF). Both made every state save/restore ++ // fail in the cosmocc build. When the model opener declines, fall back to a ++ // plain fopen-backed handle (model loads are unaffected — they succeed above). ++ if (lfile == NULL) { ++ lfile = llamafile_open_plain(fname, mode); ++ } + if (lfile == NULL) { + throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + } ++ // write_raw() writes via std::fwrite(fp); the cosmocc open path left fp NULL ++ // (read-only design). Expose the underlying FILE* so state-save writes land. ++ // Harmless for reads (read_raw_unsafe uses llamafile_read(lfile)). ++ fp = llamafile_fp(lfile); + size = llamafile_size(lfile); + return; + #endif +diff --git a/llamafile/llamafile.c b/llamafile/llamafile.c +--- a/llamafile/llamafile.c ++++ b/llamafile/llamafile.c +@@ -318,6 +318,15 @@ struct llamafile *llamafile_open_gguf(const char *fname, const char *mode) { + return llamafile_open_zip(fname, 0, mode); + } + ++// opencoti bug-269: plain fopen-backed handle for non-model files (KV state ++// save/load). llamafile_open_gguf() is model-oriented — it pread()s the GGUF ++// magic at offset 0 (which fails EBADF on a write-only "wb" fd) and routes any ++// non-GGUF file into a zip parser. State files are neither GGUF nor zip and are ++// opened for write, so they need a path with no magic probe and no zip routing. ++struct llamafile *llamafile_open_plain(const char *fname, const char *mode) { ++ return llamafile_open_file(fname, mode); ++} ++ + FILE *llamafile_fp(struct llamafile *file) { + return file->fp; + } +diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h +--- a/llamafile/llamafile.h ++++ b/llamafile/llamafile.h +@@ -46,6 +46,9 @@ extern int FLAG_verbose; // Verbose output (chatbot_main.cpp, metal.c, cu + + struct llamafile; + struct llamafile *llamafile_open_gguf(const char *, const char *); // UNUSED externally ++// opencoti bug-269: plain fopen-backed handle (no GGUF magic probe, no zip ++// routing) for non-model files such as KV state save/load. ++struct llamafile *llamafile_open_plain(const char *, const char *); + void llamafile_close(struct llamafile *); // UNUSED externally + long llamafile_read(struct llamafile *, void *, size_t); // UNUSED externally + long llamafile_write(struct llamafile *, const void *, size_t); // UNUSED externally diff --git a/patches/0072-poly-kv-pool.patch b/patches/0072-poly-kv-pool.patch new file mode 100644 index 0000000000000000000000000000000000000000..b765d7982ece62ef5d308a083859a8e4d6cb8980 --- /dev/null +++ b/patches/0072-poly-kv-pool.patch @@ -0,0 +1,98 @@ +From: opencoti +Subject: [PATCH 0072] F5 M6 PolyKV S1 — SharedKVPool shared read-only prefix + +Server-side SharedKVPool (F5 M6 S1, #370): a read-only KV prefix shared across +sequences/slots so N agents over one base context pay O(1) shared-prefix memory +instead of O(N). Per-request opt-in via the `shared_pool_slot` (>=0 selects the +pool donor slot) and `shared_prefix_n_tokens` task params; default-off (slot -1) +is byte-identical to upstream slot handling. Pure server/task plumbing — no ggml +or kernel change. + +Bench: perf/llamafile/advanced-kv-stack.bench.ts (A4/S'); gate .opencoti/m6-s1-shared-pool-gate.sh +Milestone: F5 M6 PolyKV S1 — see docs/features/poly_kv.md +Applies after: 0071 (numbered 0072 so lexical apply order lands it after the M7 + rolling-KV 0070/0071 it composes with; the original 0060 slot predates 0070). + +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -2907,6 +2907,48 @@ private: + } + } + ++ // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md ++ // F5 M6 PolyKV S1: bit-share a pool slot's read-only prefix into this seq ++ // (zero-copy cells.seq_add in unified mode) and skip re-processing it. ++ // Caller contract: this request's prompt MUST begin with exactly the pool ++ // slot's first shared_prefix_n_tokens tokens, and the pool slot must remain ++ // resident while sharers reference it. RESIDENCY PREREQUISITE: the server ++ // must be booted with --no-clear-idle. The upstream idle-slot prompt-cache ++ // offload (clear_idle, default-on with --kv-unified + --cache-ram; PR #16391) ++ // serializes an idle slot's KV to a host blob and CLEARS its device cells the ++ // moment a new task launches — which would evict the pool prefix before any ++ // sharer's seq_cp runs (seq would be empty → silent miss). We guard against ++ // that below: if the pool's resident range does not cover [0,P), we skip the ++ // share and fall through to normal full re-processing (correct, just slower). ++ // We mirror the copy_state_to() precedent: after seq_cp we also set this ++ // slot's token bookkeeping to the first P tokens, else keep_first(n_past) ++ // asserts on the (empty) slot token list (server-common.cpp:397). ++ if (slot.task->params.shared_pool_slot >= 0 && ++ slot.task->params.shared_prefix_n_tokens > 0 && ++ slot.task->params.shared_pool_slot != slot.id) { ++ const int pool = slot.task->params.shared_pool_slot; ++ int32_t P = std::min(slot.task->params.shared_prefix_n_tokens, ++ (int32_t) input_tokens.size()); ++ auto * mem = llama_get_memory(ctx_tgt); ++ // residency guard: the pool seq must actually hold cells covering [0,P) ++ const llama_pos pool_max = llama_memory_seq_pos_max(mem, pool); ++ if (pool_max < (llama_pos) (P - 1)) { ++ SLT_WRN(slot, "poly-kv-pool: pool slot %d not resident to %d (max pos %d) — " ++ "skipping share, full reprocess (boot with --no-clear-idle)\n", ++ pool, (int) (P - 1), (int) pool_max); ++ } else { ++ // clear any stale KV on this slot, then bit-share the pool's [0,P) prefix ++ llama_memory_seq_rm(mem, slot.id, -1, -1); ++ llama_memory_seq_cp(mem, pool, slot.id, 0, P); ++ // mirror token bookkeeping so keep_first(n_past)/pos_next stay consistent ++ server_tokens shared = input_tokens.clone(); ++ shared.keep_first(P); ++ slot.prompt.tokens = std::move(shared); ++ n_past = P; ++ SLT_INF(slot, "poly-kv-pool: shared %d-token prefix from slot %d (n_past -> %d)\n", (int) P, pool, n_past); ++ } ++ } ++ + // [TAG_PROMPT_LOGITS] + if (n_past == slot.task->n_tokens() && n_past > 0) { + SLT_WRN(slot, "need to evaluate at least 1 token for each active slot (n_past = %d, task.n_tokens() = %d)\n", n_past, slot.task->n_tokens()); +diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp +--- a/llama.cpp/tools/server/server-task.cpp ++++ b/llama.cpp/tools/server/server-task.cpp +@@ -273,6 +273,8 @@ task_params server_task::params_from_json_cmpl( + params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1)); + params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse); + params.session_id = json_value(data, "session_id", std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md ++ params.shared_pool_slot = json_value(data, "shared_pool_slot", -1); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md ++ params.shared_prefix_n_tokens = json_value(data, "shared_prefix_n_tokens", 0); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md + //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement + params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); + params.response_fields = json_value(data, "response_fields", std::vector()); +diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h +--- a/llama.cpp/tools/server/server-task.h ++++ b/llama.cpp/tools/server/server-task.h +@@ -72,6 +72,15 @@ struct task_params { + // number of prompt tokens before the latest user message + int32_t n_before_user = -1; + ++ // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md ++ // F5 M6 PolyKV S1 (SharedKVPool). When shared_pool_slot >= 0, the server ++ // bit-shares this request's [0, shared_prefix_n_tokens) KV with the pool ++ // slot's already-resident read-only prefix (zero-copy cells.seq_add in ++ // unified mode), so N agents share ONE physical copy of the prefix and the ++ // prefix is neither re-stored nor re-processed. <0 (default) is fully inert. ++ int32_t shared_pool_slot = -1; ++ int32_t shared_prefix_n_tokens = 0; ++ + int64_t t_max_prompt_ms = -1; // TODO: implement + int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit + diff --git a/patches/0073-turboquant-kv.patch b/patches/0073-turboquant-kv.patch new file mode 100644 index 0000000000000000000000000000000000000000..606c26d889059694eaa83265153f24ab78a3ef0c --- /dev/null +++ b/patches/0073-turboquant-kv.patch @@ -0,0 +1,3877 @@ +From: opencoti +Subject: [PATCH 0073] F5 M6 PolyKV S2 — TurboQuant KV tier family + fused FA-VEC + +TurboQuant KV (F5 M6 S2, #371/#374-396): a non-uniform Lloyd-Max centroid + +Walsh-Hadamard-rotation KV-cache quant ladder TURBO{2,3,4,8}_0 (block=group=128) +with per-channel InnerQ K-variance equalization. Three layers ship together: + - tiers + InnerQ (#371/#374-382): ggml type registration, block structs + + centroids, CPU quant/dequant (ggml-turbo-quant.c), GPU set-rows writers, the + device-symbol InnerQ coordinator (calibrate/apply, B-adapted per head_dim). + - Level-A materialize (#383-388): an InnerQ-aware turbo->f16 GPU lift on the M7 + dequant-on-lift path so any tier runs through stock f16 FA. + - Level-B fused FA-VEC (#390-396): a new op GGML_OP_TURBO_WHT rotates the small + n_q-sized Q (fwd) and FA output (inv) so flash-attention reads turbo2/3/4 K/V + IN-REGISTER (centroid x norm, WHT-rotated domain, no f16 materialize). Parseval + (WHT(Q/s).WHT(K*s)=Q.K) cancels InnerQ on the K/V read. build_attn_mha fuses + decode (n_q<=TBV_FUSE_MAX_NQ, default 8) at head_dim in {128,256}; turbo8 + + head_dim-512 keep the Level-A materialize fallback. + +Acceptance = LOGIT-EQUIVALENCE vs f16 (#355 harness), NEVER greedy byte-equality +(turboN is lossy). 4/4 tiers PASS; turbo3 mean_tv 0.0067 (tighter than q4_0's +0.0094) at 90 vs 129 MiB; fused decode scales with compression (turbo2 56 > +turbo3 51 > turbo4 48 tps @6k, breaking the Level-A flat ~41 ceiling). Composes +on the full F5 stack (gate .opencoti/m6-s3-glue-gate.sh). + +Bench: .opencoti/s2-tier-family-gate.sh, s2-tier-overview.sh, m6-s3-glue-gate.sh +Milestone: F5 M6 PolyKV S2 — see docs/features/poly_kv.md +ABI: GGML_OP_COUNT bumped by GGML_OP_TURBO_WHT (appended at end); cosmocc binary + + ggml-cuda.so DSO rebuilt and paired. +Applies after: 0071 (numbered 0073; the TurboQuant edits to fattn.cu / + llama-graph.cpp / ggml-cuda.cu sit ON TOP of 0070's rewrite of those files, so + lexical apply order MUST land this after 0070/0071; the original 0061 slot + predates 0070 being built first). + +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -17,6 +17,7 @@ LLAMA_COMMIT := $(shell cd llama.cpp 2>/dev/null && git rev-parse --short HEAD 2 + + GGML_SRCS_C := \ + llama.cpp/ggml/src/ggml-alloc.c \ ++ llama.cpp/ggml/src/ggml-turbo-quant.c \ + llama.cpp/ggml/src/ggml-quants.c \ + llama.cpp/ggml/src/ggml.c \ + llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -412,6 +412,11 @@ const std::vector kv_cache_types = { + GGML_TYPE_IQ4_NL, + GGML_TYPE_Q5_0, + GGML_TYPE_Q5_1, ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2): -ctk/-ctv turbo2|turbo3|turbo4|turbo8 ++ GGML_TYPE_TURBO2_0, ++ GGML_TYPE_TURBO3_0, ++ GGML_TYPE_TURBO4_0, ++ GGML_TYPE_TURBO8_0, + }; + + static ggml_type kv_cache_type_from_str(const std::string & s) { +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -429,7 +429,15 @@ extern "C" { + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, +- GGML_TYPE_COUNT = 42, ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2, AtomicBot fork adopt) ++ GGML_TYPE_TURBO2_0 = 42, // TurboQuant 2-bit KV: WHT + 2-bit PolarQuant ++ GGML_TYPE_TURBO3_0 = 43, // TurboQuant 3-bit KV: WHT + 3-bit PolarQuant ++ GGML_TYPE_TURBO4_0 = 44, // TurboQuant 4-bit KV: WHT + 4-bit PolarQuant ++ GGML_TYPE_TQ3_1S = 45, // TurboQuant 3-bit weight: WHT-rotated Lloyd-Max, block=32 ++ GGML_TYPE_TQ4_1S = 46, // TurboQuant 4-bit weight: WHT-rotated Lloyd-Max, block=32 ++ // opencoti-hook: turboquant tier family (M6-S2) — 8-bit near-lossless KV rung ++ GGML_TYPE_TURBO8_0 = 47, // TurboQuant 8-bit KV: WHT + uniform int8 (SCALE=256) ++ GGML_TYPE_COUNT = 48, + }; + + // precision +@@ -589,6 +597,8 @@ extern "C" { + + GGML_OP_FLASH_ATTN_TAIL_PARTIAL, // opencoti-hook: f5-rolling-kv — W4 M7 S3-a #353 + ++ GGML_OP_TURBO_WHT, // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391 ++ + GGML_OP_COUNT, + }; + +@@ -2481,6 +2491,21 @@ extern "C" { + struct ggml_tensor * a, + enum ggml_prec prec); + ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. Walsh–Hadamard ++ // rotation op for the fused turbo FA-VEC path. direction 0 = forward (Q ++ // pre-rotation: [InnerQ scale_inv] → signs1 → WHT → 1/√G·signs2); direction 1 = ++ // inverse (FA-output un-rotation: signs2 → WHT → 1/√G·signs1 → [InnerQ scale_inv]). ++ // group_size 0 ⇒ auto (128 if ne[0]%128==0 else 64). InnerQ scale is B-adapted: a ++ // is f32; pass scale=NULL and the CUDA kernel resolves InnerQ from the device ++ // symbol (turbo_innerq_active_device_scale, keyed on ne[0]); a non-NULL scale src ++ // (fork-compat) takes precedence. See docs/features/poly_kv.md. ++ GGML_API struct ggml_tensor * ggml_turbo_wht( ++ struct ggml_context * ctx, ++ struct ggml_tensor * a, ++ int direction, ++ int group_size, ++ struct ggml_tensor * scale); ++ + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + +diff --git a/llama.cpp/ggml/src/ggml-backend.cpp b/llama.cpp/ggml/src/ggml-backend.cpp +--- a/llama.cpp/ggml/src/ggml-backend.cpp ++++ b/llama.cpp/ggml/src/ggml-backend.cpp +@@ -903,6 +903,19 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st + if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) { + // since the tensor is pre-allocated, it cannot be moved to another backend + ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; ++ // OPENCOTI-DEBUG-TURBO3 (temporary, M6-S2 Gate-2): dump why no backend matched ++ fprintf(stderr, "[OPENCOTI-DBG] node=%s op=%s op->type=%s(%d) src0=%s src1=%s src2=%s\n", ++ tensor->name, ggml_op_name(tensor->op), ++ ggml_type_name(tensor->type), (int)tensor->type, ++ tensor->src[0] ? ggml_type_name(tensor->src[0]->type) : "null", ++ tensor->src[1] ? ggml_type_name(tensor->src[1]->type) : "null", ++ tensor->src[2] ? ggml_type_name(tensor->src[2]->type) : "null"); ++ for (int bi = 0; bi < sched->n_backends; bi++) { ++ const bool sb = ggml_backend_supports_buft(sched->backends[bi], buffer->buft); ++ const bool so = ggml_backend_supports_op(sched->backends[bi], tensor); ++ fprintf(stderr, "[OPENCOTI-DBG] backend[%d]=%s buft=%s supports_buft=%d supports_op=%d\n", ++ bi, ggml_backend_name(sched->backends[bi]), ggml_backend_buft_name(buffer->buft), sb, so); ++ } + GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op)); + } + +diff --git a/llama.cpp/ggml/src/ggml-common.h b/llama.cpp/ggml/src/ggml-common.h +--- a/llama.cpp/ggml/src/ggml-common.h ++++ b/llama.cpp/ggml/src/ggml-common.h +@@ -449,6 +449,113 @@ typedef struct { + } block_iq4_xs; + static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); + ++// opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2) ++// Per block: norm(fp16) + 2-bit indices (8 bytes) + 1-bit extra (4 bytes) = 14 bytes per 32 values ++// = 3.5 bits/value → 4.6× compression vs fp16 ++// The 3-bit index is split: lower 2 bits in qs[], upper 1 bit in signs[] ++#define QK_TURBO3 128 // Block size 128: one block per rotation group, eliminates redundant norms ++#define QK_TURBO3_GROUP 128 // rotation group size = head_dim ++#define QR_TURBO3 1 // opencoti-hook: turboquant (M6-S2) — dequant call emits 2 consecutive elements (float2) ++// Derived: FA template nl parameters (auto-scale with block size) ++#define NL_TURBO3 (QK_TURBO3 / 16) // non-vec FA iterations per block ++#define NL_TURBO3_VEC (QK_TURBO3 / 4) // vec FA iterations per block ++typedef struct { ++ ggml_half norm; // 2 bytes: vector L2 norm (for rescaling) ++ uint8_t qs[QK_TURBO3 / 4]; // 8 bytes: lower 2-bit indices (4 per byte) ++ uint8_t signs[QK_TURBO3 / 8]; // 4 bytes: upper 1-bit of 3-bit index (8 per byte) ++} block_turbo3_0; // 14 bytes total ++static_assert(sizeof(block_turbo3_0) == sizeof(ggml_half) + QK_TURBO3/4 + QK_TURBO3/8, "wrong turbo3_0 block size/padding"); ++ ++// TurboQuant 4-bit: 3-bit PolarQuant indices + 1-bit QJL signs ++// TURBO4_USE_4BIT: switch between 4-bit PolarQuant (new) and 3-bit+QJL (legacy) ++// Default: 4-bit on all backends (Metal + CUDA validated) ++#ifndef TURBO4_USE_4BIT ++# define TURBO4_USE_4BIT 1 ++#endif ++ ++#define QK_TURBO4 128 ++#define QR_TURBO4 1 // opencoti-hook: turboquant (M6-S2 tier family) — float2 dequant, same as turbo3 ++ ++#if TURBO4_USE_4BIT ++// 4-bit PolarQuant: 16 optimal centroids, nibble packed, no QJL ++// Per block: norm(fp16) + rnorm(fp16, reserved) + 4-bit indices (64 bytes) ++// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16 ++typedef struct { ++ ggml_half norm; // 2 bytes ++ ggml_half rnorm; // 2 bytes (reserved, unused in 4-bit mode) ++ uint8_t qs[QK_TURBO4 / 2]; // 64 bytes: 4-bit PolarQuant indices (nibble packed) ++} block_turbo4_0; // 68 bytes total ++static_assert(sizeof(block_turbo4_0) == 68, "wrong turbo4_0 block size"); ++#else ++// Legacy 3-bit PolarQuant + 1-bit QJL (original paper design) ++// Per block: norm(fp16) + rnorm(fp16) + 3-bit indices (48 bytes) + 1-bit QJL signs (16 bytes) ++// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16 ++typedef struct { ++ ggml_half norm; // 2 bytes ++ ggml_half rnorm; // 2 bytes: residual norm for QJL scale ++ uint8_t qs[QK_TURBO4 * 3 / 8]; // 48 bytes: 3-bit PolarQuant indices ++ uint8_t signs[QK_TURBO4 / 8]; // 16 bytes: 1-bit QJL signs ++} block_turbo4_0; // 68 bytes total ++static_assert(sizeof(block_turbo4_0) == 2*sizeof(ggml_half) + QK_TURBO4*3/8 + QK_TURBO4/8, "wrong turbo4_0 block size"); ++#endif ++ ++static_assert(QK_TURBO4 == 128, "turbo4 kernels assume QK_TURBO4 == 128"); ++ ++// TurboQuant 2-bit: 2-bit PolarQuant indices only (no QJL) ++// Per block: norm(fp16) + 2-bit indices (8 bytes) = 10 bytes per 32 values ++// = 2.5 bits/value → 6.4× compression vs fp16 ++// 4 centroids (Lloyd-Max for N(0, 1/128)): {-0.133462, -0.039994, 0.039994, 0.133462} ++#define QK_TURBO2 128 // Block size 128: one block per rotation group ++#define QK_TURBO2_GROUP 128 // rotation group size = head_dim ++#define QR_TURBO2 1 // opencoti-hook: turboquant (M6-S2 tier family) — float2 dequant, same as turbo3 ++// Derived: FA template nl parameters (auto-scale with block size) ++#define NL_TURBO2 (QK_TURBO2 / 16) // non-vec FA iterations per block ++#define NL_TURBO2_VEC (QK_TURBO2 / 4) // vec FA iterations per block ++typedef struct { ++ ggml_half norm; // 2 bytes: corrected L2 norm ++ uint8_t qs[QK_TURBO2 / 4]; // 8 bytes: 2-bit indices (4 per byte) ++} block_turbo2_0; // 10 bytes total ++static_assert(sizeof(block_turbo2_0) == sizeof(ggml_half) + QK_TURBO2/4, "wrong turbo2_0 block size/padding"); ++ ++// opencoti-hook: turboquant tier family (M6-S2) — TurboQuant 8-bit KV, near-lossless rung. ++// WHT-rotated, uniform signed int8 over the post-WHT normalized values (SCALE=256 ⇒ range ++// ±0.496, step 1/256). 1 byte/value, no QJL signs, no codebook. block = group = 128. ++// Per block: norm(fp16) + int8 indices (128 bytes) = 130 bytes per 128 values = 8.125 bits/value. ++#define QK_TURBO8 128 ++#define QK_TURBO8_GROUP 128 // rotation group size = head_dim ++#define QR_TURBO8 1 // float2 dequant, same convention as the other turbo tiers ++#define TURBO8_SCALE 256.0f // int8 quant scale: q = clamp(round(x*256), -127, 127) ++typedef struct { ++ ggml_half norm; // 2 bytes: corrected L2 norm ++ int8_t qs[QK_TURBO8]; // 128 bytes: signed int8 (1 per value) ++} block_turbo8_0; // 130 bytes total ++static_assert(sizeof(block_turbo8_0) == sizeof(ggml_half) + QK_TURBO8, "wrong turbo8_0 block size/padding"); ++ ++// TQ3_1S: WHT-rotated 3-bit weight quantization (8-level Lloyd-Max for N(0,1)) ++// Block size 32, dual half-block scales (d0 for [0..15], d1 for [16..31]) ++// Per block: d0(fp16) + d1(fp16) + 3-bit indices packed (12 bytes) = 16 bytes per 32 values ++// = 4.0 bits/value ++#define QK_TQ3_0 32 ++typedef struct { ++ ggml_half d0; // 2 bytes: scale for first 16 elements ++ ggml_half d1; // 2 bytes: scale for last 16 elements ++ uint8_t qs[QK_TQ3_0 * 3 / 8]; // 12 bytes: 3-bit indices packed (4 groups of 8 in 3 bytes) ++} block_tq3_1s; // 16 bytes total ++static_assert(sizeof(block_tq3_1s) == 16, "wrong tq3_1s block size"); ++ ++// TQ4_1S: WHT-rotated 4-bit weight quantization (16-level Lloyd-Max for N(0,1)) ++// Block size 32, dual half-block scales (d0 for [0..15], d1 for [16..31]) ++// Per block: d0(fp16) + d1(fp16) + 4-bit indices packed (16 bytes) = 20 bytes per 32 values ++// = 5.0 bits/value ++#define QK_TQ4_1S 32 ++typedef struct { ++ ggml_half d0; // 2 bytes: scale for first 16 elements ++ ggml_half d1; // 2 bytes: scale for last 16 elements ++ uint8_t qs[QK_TQ4_1S / 2]; // 16 bytes: 4-bit indices nibble-packed ++} block_tq4_1s; // 20 bytes total ++static_assert(sizeof(block_tq4_1s) == 20, "wrong tq4_1s block size"); ++ ++ + #endif // GGML_COMMON_DECL + #endif // GGML_COMMON_DECL + +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -412,6 +412,27 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { + [GGML_TYPE_I32] = { + .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32, + }, ++ // opencoti-hook: turboquant (M6-S2) — turbo3 KV tail write needs a CPU from_float. In ++ // POSITION_WINDOW mode the host-pinned KV tail set_rows runs on the CPU backend; ++ // ggml_compute_forward_set_rows (ggml-cpu/ops.cpp) calls type_traits_cpu[dst->type].from_float, ++ // which was NULL for turbo3 → call 0x0 → PC=0 SIGSEGV at warmup (512-tok ubatch overflows the ++ // 256-cell window). quantize_row_turbo3_0_ref is the same forward-WHT+3bit+corrected-norm as the ++ // GPU set_rows kernel, so window/tail stay format-consistent for the dequant-on-lift. See poly_kv.md. ++ [GGML_TYPE_TURBO3_0] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo3_0_ref, ++ }, ++ // opencoti-hook: turboquant tier family (M6-S2) — same CPU from_float requirement as turbo3 ++ // for POSITION_WINDOW host-pinned tail set_rows. forward-WHT + Nbit + corrected-norm matches ++ // the GPU k_set_rows_turboN kernels ⇒ window/tail format-consistent for the dequant-on-lift. ++ [GGML_TYPE_TURBO4_0] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo4_0_ref, ++ }, ++ [GGML_TYPE_TURBO2_0] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo2_0_ref, ++ }, ++ [GGML_TYPE_TURBO8_0] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo8_0_ref, ++ }, + }; + + const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) { +@@ -1753,6 +1774,16 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + return; + } + ++ // opencoti diag (M6-S2): env-gated per-op trace to localize the turbo3 CPU crash; remove before ship. ++ if (params->ith == 0 && getenv("OPENCOTI_OPTRACE")) { ++ fprintf(stderr, "[OPTRACE] op=%s name=%s dst=%s s0=%s s1=%s\n", ++ ggml_op_name(tensor->op), tensor->name, ++ ggml_type_name(tensor->type), ++ tensor->src[0] ? ggml_type_name(tensor->src[0]->type) : "-", ++ tensor->src[1] ? ggml_type_name(tensor->src[1]->type) : "-"); ++ fflush(stderr); ++ } ++ + // extra_buffer op? + if (ggml_cpu_extra_compute_forward(params, tensor)) { + return; +@@ -2089,6 +2120,15 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + // engine), a different path entirely. + GGML_ABORT("STREAMING_FLASH_ATTN has no CPU implementation (CUDA-only op)"); + } break; ++ case GGML_OP_TURBO_WHT: ++ { ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. ++ // CUDA-only op. The build_attn_mha fused-turbo gate emits it only on ++ // the GPU-resident turbo FA-VEC path (head_dim ≤ 256, K/V resident on ++ // CUDA), so — like STREAMING_FLASH_ATTN — the CPU scheduler never ++ // reaches here. supports_op keeps it on the CUDA backend. ++ GGML_ABORT("TURBO_WHT has no CPU implementation (CUDA-only op)"); ++ } break; + case GGML_OP_GET_REL_POS: + { + ggml_compute_forward_get_rel_pos(params, tensor); +@@ -2368,6 +2408,13 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + // scheduled on CPU. Single task keeps the planner well-formed. + n_tasks = 1; + } break; ++ case GGML_OP_TURBO_WHT: ++ { ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. ++ // CUDA-only; never scheduled on CPU. Single task keeps the planner ++ // well-formed. ++ n_tasks = 1; ++ } break; + case GGML_OP_SILU_BACK: + case GGML_OP_MUL: + case GGML_OP_DIV: +diff --git a/llama.cpp/ggml/src/ggml-cpu/quants.h b/llama.cpp/ggml/src/ggml-cpu/quants.h +--- a/llama.cpp/ggml/src/ggml-cpu/quants.h ++++ b/llama.cpp/ggml/src/ggml-cpu/quants.h +@@ -19,6 +19,11 @@ void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in + void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); ++// opencoti-hook: turboquant (M6-S2) — turbo CPU from_float for POSITION_WINDOW host-tail KV write; see docs/features/poly_kv.md ++void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * GGML_RESTRICT y, int64_t k); ++void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k); ++void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k); ++void quantize_row_turbo8_0_ref(const float * GGML_RESTRICT x, block_turbo8_0 * GGML_RESTRICT y, int64_t k); + + void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +diff --git a/llama.cpp/ggml/src/ggml-cuda/convert.cu b/llama.cpp/ggml/src/ggml-cuda/convert.cu +--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu +@@ -1,5 +1,6 @@ + #include "convert.cuh" + #include "dequantize.cuh" ++#include "turbo-dequant.cuh" // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2) + + #include + +@@ -726,6 +727,15 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { + return dequantize_block_q8_0_f16_cuda; + } + return dequantize_block_cont_cuda; ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2): inverse-WHT dequant → TRUE-space f16 ++ case GGML_TYPE_TURBO3_0: ++ return dequantize_block_cont_cuda; ++ case GGML_TYPE_TURBO4_0: ++ return dequantize_block_cont_cuda; ++ case GGML_TYPE_TURBO2_0: ++ return dequantize_block_cont_cuda; ++ case GGML_TYPE_TURBO8_0: ++ return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_K: + return dequantize_row_q2_K_cuda; + case GGML_TYPE_Q3_K: +@@ -842,6 +852,15 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { + return dequantize_block_cuda; + case GGML_TYPE_Q8_0: + return dequantize_block_cuda; ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2): S3d lift path → TRUE-space f16 ++ case GGML_TYPE_TURBO3_0: ++ return dequantize_block_cuda; ++ case GGML_TYPE_TURBO4_0: ++ return dequantize_block_cuda; ++ case GGML_TYPE_TURBO2_0: ++ return dequantize_block_cuda; ++ case GGML_TYPE_TURBO8_0: ++ return dequantize_block_cuda; + case GGML_TYPE_BF16: + return convert_unary_cuda; + default: +diff --git a/llama.cpp/ggml/src/ggml-cuda/cpy.cu b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +@@ -1,6 +1,8 @@ + #include "cpy.cuh" + #include "dequantize.cuh" + #include "cpy-utils.cuh" ++#include "turbo-cpy.cuh" // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — GPU dequant-on-lift ++#include "turbo-innerq-dev.cuh" // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — InnerQ device scale accessor + #if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY) + #include "ggml-musa/mudnn.cuh" + #endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY +@@ -558,6 +560,20 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } ++ // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — GPU-resident turbo→f16 dequant-on-lift. ++ // Replaces the CPU-spill cast (the scheduler used to assign ggml_cast(turboN→f16) to the CPU ++ // backend because CUDA CPY supports_op rejected turbo). The InnerQ device scale (owned by ++ // set-rows.cu) is fetched by the cast's row width (ne00 = n_embd_gqa); nullptr ⇒ identity ++ // (default-off / no slot). See docs/features/poly_kv.md (M6-S2 perf-path Level A). ++ } else if ((src0->type == GGML_TYPE_TURBO2_0 || src0->type == GGML_TYPE_TURBO3_0 || ++ src0->type == GGML_TYPE_TURBO4_0 || src0->type == GGML_TYPE_TURBO8_0) && ++ src1->type == GGML_TYPE_F16) { ++ const float * d_scale_inv = nullptr; ++ int head_dim = 0; ++ turbo_innerq_active_device_scale(ne00, &d_scale_inv, &head_dim); ++ ggml_cpy_turbo_f16_cuda(src0->type, src0_ddc, src1_ddc, ne, ne00, ne01, ne02, ++ nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, ++ d_scale_inv, head_dim, main_stream); + } else { + GGML_ABORT("%s: unsupported type combination (%s to %s)\n", __func__, + ggml_type_name(src0->type), ggml_type_name(src1->type)); +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -584,10 +584,21 @@ static __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict + } + } + ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. In-register turbo K/V readers ++// (centroid×norm, rotated space) for the fused FA-VEC path. Included here so get_vec_dot_KQ / ++// get_dequantize_V below can dispatch turbo2/3/4. ggml_cuda_mad + block_turbo* are already in scope. ++#include "fattn-turbo.cuh" ++ + template + constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() { + if constexpr (type_K == GGML_TYPE_F16) { + return vec_dot_fattn_vec_KQ_f16; ++ } else if constexpr (type_K == GGML_TYPE_TURBO2_0) { // opencoti-hook: turboquant perf-lift (Level B) ++ return vec_dot_fattn_vec_KQ_turbo2; ++ } else if constexpr (type_K == GGML_TYPE_TURBO3_0) { ++ return vec_dot_fattn_vec_KQ_turbo3; ++ } else if constexpr (type_K == GGML_TYPE_TURBO4_0) { ++ return vec_dot_fattn_vec_KQ_turbo4; + } else if constexpr (type_K == GGML_TYPE_Q4_0) { + return vec_dot_fattn_vec_KQ_q4_0; + } else if constexpr (type_K == GGML_TYPE_Q4_1) { +@@ -610,6 +621,12 @@ template + constexpr __device__ dequantize_V_t get_dequantize_V() { + if constexpr (type_V == GGML_TYPE_F16) { + return dequantize_V_f16; ++ } else if constexpr (type_V == GGML_TYPE_TURBO2_0) { // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 ++ return dequantize_V_turbo2; ++ } else if constexpr (type_V == GGML_TYPE_TURBO3_0) { ++ return dequantize_V_turbo3; ++ } else if constexpr (type_V == GGML_TYPE_TURBO4_0) { ++ return dequantize_V_turbo4; + } else if constexpr (type_V == GGML_TYPE_Q4_0) { + return dequantize_V_q4_0; + } else if constexpr (type_V == GGML_TYPE_Q4_1) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh +@@ -0,0 +1,129 @@ ++// opencoti-hook: turboquant perf-lift (Level B) — see docs/features/poly_kv.md (F5 M6-S2 #392) ++// ++// In-register turbo K/V readers for the FUSED FA-VEC path. The turbo K/V cache stays compressed ++// AND ROTATED on the GPU; these readers reconstruct centroid×norm (the WHT-rotated value) with NO ++// inverse WHT — the Parseval identity makes Q_rot·K_rot == Q_true·K_true (Q is forward-rotated by ++// the L3 ggml_turbo_wht(Q,0)), and the rotated attention output is un-rotated once by ++// ggml_turbo_wht(O,1) at L3. InnerQ is therefore NOT applied here: it folds entirely into the Q ++// pre-rotation (×scale_inv) and the output inverse-WHT (×scale_inv) — K and V reads are pure ++// centroid×norm, exactly as the AtomicBot fork does. ++// ++// Turbo K is treated like F16 in our newer FA-VEC kernel: Q is kept FLOAT (Q_q8_1=false), and the ++// turbo vec_dot mirrors vec_dot_fattn_vec_KQ_f16 — the per-half2 memcpy load becomes a centroid ++// decode. A half2 pair (elements 2*e2, 2*e2+1) never straddles a 128-block (even start), and the ++// V dequant i0 is 4-aligned, so block math is branch-free. Centroid tables are byte-identical to ++// turbo-dequant.cuh / turbo-cpy.cu (validated in Level A). turbo8 / head_dim 512 are NOT here — ++// they fall back to the Level-A materialize lift (fork has no turbo8, no D=512 vec instance). ++#pragma once ++ ++#include "common.cuh" ++ ++// ---- centroid codebooks (== turbo-dequant.cuh TURBO{2,3,4}_CENTROIDS; N(0,1/128) Lloyd-Max) ---- ++static __constant__ float FATTN_TURBO3_CENTROIDS[8] = { ++ -0.190685f, -0.117832f, -0.065717f, -0.021460f, 0.021460f, 0.065717f, 0.117832f, 0.190685f }; ++static __constant__ float FATTN_TURBO4_CENTROIDS[16] = { ++ -0.173926f, -0.117195f, -0.089527f, -0.068756f, -0.051262f, -0.035597f, -0.020989f, -0.006938f, ++ 0.006938f, 0.020989f, 0.035597f, 0.051262f, 0.068756f, 0.089527f, 0.117195f, 0.173926f }; ++static __constant__ float FATTN_TURBO2_CENTROIDS[4] = { ++ -0.133462f, -0.039994f, 0.039994f, 0.133462f }; ++ ++// ---- per-element rotated-value decode (centroid[code]×norm). p = position within the 128-block. ---- ++static __device__ __forceinline__ float fattn_turbo3_val(const block_turbo3_0 * blk, int p, float norm) { ++ const uint8_t lo = (blk->qs[p >> 2] >> ((p & 3) * 2)) & 0x3; ++ const uint8_t hi = (blk->signs[p >> 3] >> (p & 7)) & 0x1; ++ return FATTN_TURBO3_CENTROIDS[lo | (hi << 2)] * norm; ++} ++static __device__ __forceinline__ float fattn_turbo4_val(const block_turbo4_0 * blk, int p, float norm) { ++ const uint8_t idx = (blk->qs[p >> 1] >> ((p & 1) * 4)) & 0xF; ++ return FATTN_TURBO4_CENTROIDS[idx] * norm; ++} ++static __device__ __forceinline__ float fattn_turbo2_val(const block_turbo2_0 * blk, int p, float norm) { ++ const uint8_t idx = (blk->qs[p >> 2] >> ((p & 3) * 2)) & 0x3; ++ return FATTN_TURBO2_CENTROIDS[idx] * norm; ++} ++ ++// Q·K MAD against the float Q slice — Q_reg is half2 with V_DOT2_F32_F16_AVAILABLE, else float2 ++// (mirrors vec_dot_fattn_vec_KQ_f16). kval is the decoded rotated-K half2 for this lane pair. ++static __device__ __forceinline__ void fattn_turbo_qmad(float & sum, half2 kval, const void * Q_v, int qi) { ++#ifdef V_DOT2_F32_F16_AVAILABLE ++ ggml_cuda_mad(sum, kval , ((const half2 *) Q_v)[qi]); ++#else ++ ggml_cuda_mad(sum, __half22float2(kval), ((const float2 *) Q_v)[qi]); ++#endif // V_DOT2_F32_F16_AVAILABLE ++} ++ ++// ===================== K: vec_dot (centroid×norm · float Q), f16-structured ===================== ++// K_c → key row in turbo format; Q_v → this lane's float Q slice (Q_reg, already ×scale); the dot ++// covers this thread's D-slice, warp_reduce_sum combines lanes (done by the caller). ++#define FATTN_TURBO_KQ_BODY(BLOCK_T, VAL_FN) \ ++ GGML_UNUSED(Q_q8); GGML_UNUSED(Q_ds_v); \ ++ constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes(); \ ++ constexpr int cpy_ne = cpy_nb / 4; \ ++ float sum = 0.0f; \ ++ _Pragma("unroll") \ ++ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) { \ ++ half2 tmp[cpy_ne]; \ ++ _Pragma("unroll") \ ++ for (int k1 = 0; k1 < cpy_ne; ++k1) { \ ++ const int e2 = k_KQ_0 + (int)(threadIdx.x % nthreads)*cpy_ne + k1; \ ++ const int e = 2*e2; \ ++ const BLOCK_T * blk = (const BLOCK_T *) K_c + (e >> 7); \ ++ const float norm = __half2float(blk->norm); \ ++ const int ib = e & 127; \ ++ tmp[k1] = make_half2(VAL_FN(blk, ib, norm), VAL_FN(blk, ib + 1, norm)); \ ++ } \ ++ _Pragma("unroll") \ ++ for (int k1 = 0; k1 < cpy_ne; ++k1) { \ ++ fattn_turbo_qmad(sum, tmp[k1], Q_v, k_KQ_0/nthreads + k1); \ ++ } \ ++ } \ ++ return sum; ++ ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ FATTN_TURBO_KQ_BODY(block_turbo3_0, fattn_turbo3_val) ++} ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo4( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ FATTN_TURBO_KQ_BODY(block_turbo4_0, fattn_turbo4_val) ++} ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ FATTN_TURBO_KQ_BODY(block_turbo2_0, fattn_turbo2_val) ++} ++ ++// ===================== V: dequant (centroid×norm), ne consecutive (i0 4-aligned) ===================== ++// Reads ne consecutive rotated V values at element offset i0; writes T (half/float) as half2/float2. ++#define FATTN_TURBO_V_BODY(BLOCK_T, VAL_FN) \ ++ static_assert(ne % 2 == 0, "bad ne"); \ ++ const BLOCK_T * blk = (const BLOCK_T *) vx + (i0 >> 7); \ ++ const float norm = __half2float(blk->norm); \ ++ const int ib = (int)(i0 & 127); \ ++ float vals[ne]; \ ++ _Pragma("unroll") \ ++ for (int l = 0; l < ne; ++l) vals[l] = VAL_FN(blk, ib + l, norm); \ ++ if constexpr (std::is_same_v) { \ ++ half2 * d2 = (half2 *) dst; \ ++ _Pragma("unroll") \ ++ for (int l = 0; l < ne; l += 2) d2[l/2] = make_half2(vals[l], vals[l+1]); \ ++ } else { \ ++ float2 * d2 = (float2 *) dst; \ ++ _Pragma("unroll") \ ++ for (int l = 0; l < ne; l += 2) d2[l/2] = make_float2(vals[l], vals[l+1]); \ ++ } ++ ++template ++static __device__ __forceinline__ void dequantize_V_turbo3(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ FATTN_TURBO_V_BODY(block_turbo3_0, fattn_turbo3_val) ++} ++template ++static __device__ __forceinline__ void dequantize_V_turbo4(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ FATTN_TURBO_V_BODY(block_turbo4_0, fattn_turbo4_val) ++} ++template ++static __device__ __forceinline__ void dequantize_V_turbo2(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ FATTN_TURBO_V_BODY(block_turbo2_0, fattn_turbo2_val) ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -75,8 +75,12 @@ static __global__ void flash_attn_ext_vec( + constexpr int nthreads_V_q = (D/4 < 32 ? D/4 : 32); + #endif // GGML_USE_HIP + ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Turbo K reads centroid×norm ++ // against a FLOAT Q (Q_q8_1=false below) and its vec_dot mirrors the f16 path, so it takes the ++ // f16-style KQ thread split. Turbo V uses the quantized split (nthreads_V_q, V_rows_per_thread=4). ++ constexpr bool is_turbo_K = (type_K == GGML_TYPE_TURBO2_0 || type_K == GGML_TYPE_TURBO3_0 || type_K == GGML_TYPE_TURBO4_0); + constexpr int nthreads = ggml_cuda_fattn_vec_get_nthreads_device(); +- constexpr int nthreads_KQ = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_KQ_q; ++ constexpr int nthreads_KQ = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16 || is_turbo_K) ? 128 / cpy_nb : nthreads_KQ_q; + constexpr int nthreads_V = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_V_q; + + static_assert(WARP_SIZE % nthreads_KQ == 0, "bad nthreads_K"); +@@ -86,7 +90,9 @@ static __global__ void flash_attn_ext_vec( + constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V; + + constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ(); +- constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16; ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Keep Q FLOAT for turbo K ++ // (q8_1-quantizing Q would destroy the WHT-rotation precision the Parseval identity needs). ++ constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16 && !is_turbo_K; + #ifdef V_DOT2_F32_F16_AVAILABLE + constexpr dequantize_V_t dequantize_V = get_dequantize_V(); + #else +@@ -601,3 +607,14 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) ++ ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec instances ++// (K==V same tier only; instances in template-instances/fattn-vec-instance-turbo{2,3,4}_0-*.cu). ++#define EXTERN_DECL_FATTN_VEC_TURBO(type_T) \ ++ extern DECL_FATTN_VEC_CASE( 64, type_T, type_T); \ ++ extern DECL_FATTN_VEC_CASE(128, type_T, type_T); \ ++ extern DECL_FATTN_VEC_CASE(256, type_T, type_T); \ ++ ++EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_0) ++EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_0) ++EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO4_0) +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -330,6 +330,13 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) + #endif // GGML_CUDA_FA_ALL_QUANTS + ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec path ++ // (always registered, independent of GGML_CUDA_FA_ALL_QUANTS). turbo2/3/4 at head_dim ∈ ++ // {64,128,256}; turbo8 / head_dim 512 are NOT here (Level-A materialize lift handles them). ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0) ++ + GGML_ABORT("fatal error"); + } + +@@ -446,6 +453,10 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + case GGML_TYPE_Q8_0: + case GGML_TYPE_BF16: + break; ++ case GGML_TYPE_TURBO2_0: // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 ++ case GGML_TYPE_TURBO3_0: ++ case GGML_TYPE_TURBO4_0: ++ break; + default: + return BEST_FATTN_KERNEL_NONE; + } +@@ -458,6 +469,14 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path. + const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0; + ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Turbo K/V have ONLY a fused ++ // VEC kernel (centroid×norm read in-register); never route them to MMA/WMMA/TILE, which can't ++ // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when ++ // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift. ++ if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0) { ++ return can_use_vector_kernel ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE; ++ } ++ + // If Turing tensor cores are available, use them: + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -68,6 +68,7 @@ + #include "ggml-cuda/tri.cuh" + #include "ggml-cuda/cumsum.cuh" + #include "ggml-cuda/fill.cuh" ++#include "ggml-cuda/turbo-wht.cuh" // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391 + #include "ggml.h" + + #include +@@ -3039,6 +3040,13 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg + // here in F32, not delegated to the kernel. See fattn.cu. + ggml_cuda_streaming_flash_attn(ctx, dst); + break; ++ case GGML_OP_TURBO_WHT: ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. ++ // Walsh–Hadamard rotation of the small Q (fwd) / FA-output (inv) tensors ++ // for the fused turbo FA-VEC path. InnerQ resolved device-side. See ++ // turbo-wht.cu. ++ ggml_cuda_turbo_wht(ctx, dst); ++ break; + case GGML_OP_OUT_PROD: + ggml_cuda_out_prod(ctx, dst); + break; +@@ -5375,9 +5383,13 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + } break; + case GGML_OP_SET_ROWS: + { ++ // opencoti-hook: turboquant tier family — see docs/features/poly_kv.md (M6-S2): ++ // accept turbo2/turbo3/turbo4 dst (each has a GPU k_set_rows_turboN write kernel). + return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || +- op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && ++ op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL || ++ op->type == GGML_TYPE_TURBO3_0 || op->type == GGML_TYPE_TURBO4_0 || ++ op->type == GGML_TYPE_TURBO2_0 || op->type == GGML_TYPE_TURBO8_0) && + op->src[0]->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); + } break; +@@ -5397,6 +5409,15 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + ) { + return true; + } ++ // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — GPU-resident turbo→f16 ++ // dequant-on-lift (ggml_cuda_cpy → ggml_cpy_turbo_f16_cuda). Lets the scheduler keep ++ // build_attn_mha's ggml_cast(turboN→f16) on CUDA instead of spilling to the CPU ++ // backend. See docs/features/poly_kv.md (M6-S2 perf-path Level A). ++ if ((src0_type == GGML_TYPE_TURBO2_0 || src0_type == GGML_TYPE_TURBO3_0 || ++ src0_type == GGML_TYPE_TURBO4_0 || src0_type == GGML_TYPE_TURBO8_0) && ++ src1_type == GGML_TYPE_F16) { ++ return true; ++ } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) { + return true; + } +@@ -5623,6 +5644,18 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + return false; + } + } ++ case GGML_OP_TURBO_WHT: { ++ // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. f32 in/out, ++ // contiguous, head_dim (ne[0]) a multiple of the 128 WHT group. group_size is ++ // stored in op_params[1] (the ctor forces 128 for our path). InnerQ resolved ++ // device-side; no src[1] requirement. ++ const ggml_tensor * a = op->src[0]; ++ if (a->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) return false; ++ if (!ggml_is_contiguous(a) || !ggml_is_contiguous(op)) return false; ++ int group_size = 0; ++ memcpy(&group_size, (const char *) op->op_params + sizeof(int), sizeof(int)); ++ return group_size == 128 && a->ne[0] % 128 == 0; ++ } + case GGML_OP_CROSS_ENTROPY_LOSS: + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + case GGML_OP_OPT_STEP_ADAMW: +diff --git a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +--- a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +@@ -1,5 +1,10 @@ + #include "set-rows.cuh" + #include "cpy-utils.cuh" ++#include "turbo-dequant.cuh" // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2 write path) ++#include "turbo-innerq.cuh" // opencoti-hook: turboquant InnerQ (M6-S2) — per-channel equalization ++#include "turbo-innerq-dev.cuh" // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — cross-TU device-scale accessor ++#include ++#include + + typedef void (*set_rows_kernel_t)(const char * src, char * dst); + +@@ -214,6 +219,801 @@ static void set_rows_cuda( + } + } + ++// opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2 write path). ++// f32 K/V → turbo3: group-level forward WHT + 3-bit centroid quantize + corrected norm. ++// One block per 128-element WHT group (QK_TURBO3 == group). InnerQ stub-disabled ++// (Gate-3 re-enables). No tail kernel — head_dim must be 128-block-aligned (Qwen 128, ++// Gemma 256 = 2x128). Algorithm + warp/ballot packing ported from the AtomicBot fork's ++// ggml-cuda/set-rows.cu k_set_rows_turbo3, with the InnerQ calibrate/apply blocks removed. ++// ===== opencoti-hook: turboquant InnerQ (M6-S2) — per-channel equalization ===== ++// Device state — set-rows.cu is the ONLY TU with device access (the Qwen gate uses ++// the host CPU dequant; cross-TU GPU dequant InnerQ for the iSWA streaming path is a ++// deferred follow-up). __device__ globals are zero-initialized ⇒ OFF mode is inert. ++// opencoti-hook: turboquant InnerQ (M6-S2) — PER-LAYER (per-width) head_dim folding. ++// ne00 of the KV set-rows row is n_embd_gqa = head_dim*n_head_kv (NOT head_dim). A model ++// can mix head_dims across layers: Gemma-4 = 256-SWA (n_embd_gqa 256*8=2048) + 512-global ++// (512*2=1024). head_dim is NOT derivable from ne00 alone (ambiguous), and is unavailable ++// at the CUDA launch site, so the host supplies a {n_embd_gqa→head_dim} map via env ++// (TURBO_INNERQ_HEADDIM_MAP="2048:256,1024:512"). Each distinct width is a SLOT with its ++// own scale_inv[head_dim] and sq_accum — no cross-width contamination. The dispatch maps ++// ne00→slot and passes it as a kernel arg (no per-launch H2D). The per-head channel is ++// (i_grp*128 + j) % head_dim[slot]; scale shared across heads of that width. head_dim==128 ++// ⇒ fold is identity (j). Empty map ⇒ one wildcard slot (back-compat, uniform models). ++static __device__ float d_innerq_scale [INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++static __device__ float d_innerq_scale_inv[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++static __device__ float d_innerq_sq_accum [INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++static __device__ int d_innerq_slot_hd [INNERQ_MAX_SLOTS]; // head_dim per slot ++static __device__ int d_innerq_count; // total groups seen (finalize trigger) ++static __device__ int d_innerq_active; // 0 = identity, 1 = scales applied ++static __device__ int d_innerq_calibrating; // 1 = accumulating K² stats ++ ++static int innerq_mode = 0; // 0=off, 1=calibrating, 2=active ++static int innerq_target_blocks = 0; ++static float innerq_strength = 0.5f; ++static bool innerq_initialized = false; ++static const char * innerq_out_path = nullptr; ++// Per-width slot table (host mirror of the device map). slot_nembd[s]==0 ⇒ wildcard. ++static int innerq_nslots = 0; ++static int innerq_slot_nembd[INNERQ_MAX_SLOTS] = {0}; ++static int innerq_slot_hd [INNERQ_MAX_SLOTS] = {0}; ++ ++// Map a set-rows row width (ne00 = n_embd_gqa) to its InnerQ slot, or -1 (no InnerQ). ++// Exact width match first, then a wildcard (nembd==0) slot for uniform/back-compat. ++static int innerq_slot_for(int ne00) { ++ for (int s = 0; s < innerq_nslots; s++) if (innerq_slot_nembd[s] == ne00) return s; ++ for (int s = 0; s < innerq_nslots; s++) if (innerq_slot_nembd[s] == 0) return s; ++ return -1; ++} ++ ++// Parse TURBO_INNERQ_HEADDIM_MAP="ne0:hd,ne1:hd,...". Returns nslots (0 if unset/empty). ++// On empty/parse-fail the caller falls back to a single wildcard slot from HEAD_DIM. ++static int innerq_parse_map(int * nembd, int * hd) { ++ const char * m = getenv("TURBO_INNERQ_HEADDIM_MAP"); ++ if (!m || !m[0]) return 0; ++ int n = 0; ++ const char * p = m; ++ while (*p && n < INNERQ_MAX_SLOTS) { ++ char * end = nullptr; ++ long ne = strtol(p, &end, 10); ++ if (end == p || *end != ':') break; ++ p = end + 1; ++ long h = strtol(p, &end, 10); ++ if (end == p) break; ++ if (ne > 0 && h >= 128 && h <= INNERQ_MAX_CHANNELS && (h % 128) == 0) { ++ nembd[n] = (int) ne; hd[n] = (int) h; n++; ++ } ++ p = end; ++ while (*p == ',' || *p == ' ') p++; ++ } ++ return n; ++} ++ ++// ACTIVE-from-token-0: upload per-slot scale (=1/scale_inv) + scale_inv + head_dim, ++// publish the host slot table, activate. scale_inv_in[s][0..hd[s]). ++static void turbo_innerq_upload_active(int nslots, const int * nembd, const int * hd, ++ const float scale_inv_in[][INNERQ_MAX_CHANNELS]) { ++ float scale[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++ float sinv [INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++ int hd_dev[INNERQ_MAX_SLOTS]; ++ for (int s = 0; s < INNERQ_MAX_SLOTS; s++) { ++ const int h = (s < nslots) ? hd[s] : 0; ++ hd_dev[s] = h; ++ for (int i = 0; i < INNERQ_MAX_CHANNELS; i++) { ++ const float si = (s < nslots && i < h) ? scale_inv_in[s][i] : 1.0f; ++ sinv [s][i] = si; ++ scale[s][i] = (si > 1e-20f) ? (1.0f / si) : 1.0f; ++ } ++ } ++ // publish host mirror (slot selection at dispatch) ++ innerq_nslots = nslots; ++ for (int s = 0; s < INNERQ_MAX_SLOTS; s++) { ++ innerq_slot_nembd[s] = (s < nslots) ? nembd[s] : 0; ++ innerq_slot_hd[s] = (s < nslots) ? hd[s] : 0; ++ } ++ int zero = 0, one = 1; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_scale, scale, sizeof(scale))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_scale_inv, sinv, sizeof(sinv))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_slot_hd, hd_dev, sizeof(hd_dev))); ++ CUDA_CHECK(cudaDeviceSynchronize()); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_active, &one, sizeof(int))); ++} ++ ++static void turbo_innerq_init(void) { ++ if (innerq_initialized) return; ++ innerq_initialized = true; ++ ++ const char * scale_path = getenv("TURBO_INNERQ_SCALE"); ++ if (scale_path && scale_path[0]) { ++ static float sinv[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++ int nembd[INNERQ_MAX_SLOTS], hd[INNERQ_MAX_SLOTS]; ++ const int ns = turbo_innerq_load_file_v2(scale_path, nembd, hd, sinv); ++ if (ns > 0) { ++ turbo_innerq_upload_active(ns, nembd, hd, sinv); ++ innerq_mode = 2; ++ GGML_LOG_INFO("turbo_innerq_init: InnerQ ACTIVE from token 0 (%d slot(s) from %s)\n", ns, scale_path); ++ for (int s = 0; s < ns; s++) ++ GGML_LOG_INFO("turbo_innerq_init: slot %d: n_embd_gqa=%d head_dim=%d\n", s, nembd[s], hd[s]); ++ } else { ++ GGML_LOG_WARN("turbo_innerq_init: InnerQ scale file %s unreadable — disabled\n", scale_path); ++ } ++ return; ++ } ++ ++ const char * cal = getenv("TURBO_INNERQ_CALIBRATE"); ++ if (cal && atoi(cal) > 0) { ++ innerq_target_blocks = atoi(cal); ++ innerq_out_path = getenv("TURBO_INNERQ_OUT"); ++ const char * str = getenv("TURBO_INNERQ_STRENGTH"); ++ if (str) innerq_strength = (float) atof(str); ++ if (innerq_strength <= 0.0f || innerq_strength > 1.0f) innerq_strength = 0.5f; ++ // Per-width slots: TURBO_INNERQ_HEADDIM_MAP="2048:256,1024:512" (Gemma mixed). For ++ // uniform models leave it unset and pass TURBO_INNERQ_HEAD_DIM (default 128) ⇒ one ++ // wildcard slot (n_embd_gqa=0 matches any width). Each slot folds per its head_dim. ++ innerq_nslots = innerq_parse_map(innerq_slot_nembd, innerq_slot_hd); ++ if (innerq_nslots <= 0) { ++ const char * hd_env = getenv("TURBO_INNERQ_HEAD_DIM"); ++ int hd = (hd_env && hd_env[0]) ? atoi(hd_env) : 128; ++ if (hd < 128 || hd > INNERQ_MAX_CHANNELS || (hd % 128) != 0) hd = 128; ++ innerq_nslots = 1; innerq_slot_nembd[0] = 0; innerq_slot_hd[0] = hd; ++ } ++ int slot_hd_dev[INNERQ_MAX_SLOTS] = {0}; ++ for (int s = 0; s < innerq_nslots; s++) slot_hd_dev[s] = innerq_slot_hd[s]; ++ float zeros[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS] = {{0}}; ++ int zero = 0, one = 1; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_sq_accum, zeros, sizeof(zeros))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_count, &zero, sizeof(int))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_active, &zero, sizeof(int))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_slot_hd, slot_hd_dev, sizeof(slot_hd_dev))); ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_calibrating, &one, sizeof(int))); ++ innerq_mode = 1; ++ GGML_LOG_INFO("turbo_innerq_init: InnerQ CALIBRATING (target=%d blocks, strength=%.2f, %d slot(s), out=%s)\n", ++ innerq_target_blocks, innerq_strength, innerq_nslots, innerq_out_path ? innerq_out_path : "(none)"); ++ for (int s = 0; s < innerq_nslots; s++) ++ GGML_LOG_INFO("turbo_innerq_init: slot %d: n_embd_gqa=%d head_dim=%d\n", s, innerq_slot_nembd[s], innerq_slot_hd[s]); ++ } ++} ++ ++// opencoti-hook: turboquant perf-lift (M6-S2 Level A) — cross-TU device-scale accessor. ++// Exposes the active slot's scale_inv device row + head_dim to the GPU dequant-on-lift cpy kernel ++// (turbo-cpy.cu), selecting the slot by the cast's row width (ne00 = n_embd_gqa) exactly as the CPU ++// dequant does (turbo_innerq_cpu_slot_for). Returns false when InnerQ is not ACTIVE or no slot ++// matches ⇒ the kernel un-equalizes with identity (no ×scale_inv), matching the CPU miss path. ++bool turbo_innerq_active_device_scale(int64_t ne00, const float ** d_scale_inv_out, int * head_dim_out) { ++ turbo_innerq_init(); // idempotent (innerq_initialized guard) ++ if (innerq_mode != 2) return false; // 2 = ACTIVE (not off/calibrating) ++ const int slot = innerq_slot_for((int) ne00); ++ if (slot < 0) return false; ++ void * base = nullptr; ++ if (cudaGetSymbolAddress(&base, d_innerq_scale_inv) != cudaSuccess || base == nullptr) return false; ++ *d_scale_inv_out = (const float *) base + (size_t) slot * INNERQ_MAX_CHANNELS; ++ *head_dim_out = innerq_slot_hd[slot]; ++ return true; ++} ++ ++static void turbo_innerq_finalize(int group_size) { ++ // Per-slot finalize. Within a slot, every head-channel is written the same number of ++ // times (n_head_kv*n_tokens*n_layers_of_that_width), so that uniform count cancels in ++ // the mean_rms/rms ratio — no per-slot count normalization needed (a single global ++ // `count` is only the finalize trigger). Slots are independent (no cross-width mixing). ++ (void) group_size; ++ float sq_accum[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++ int count = 0; ++ CUDA_CHECK(cudaMemcpyFromSymbol(sq_accum, d_innerq_sq_accum, sizeof(sq_accum))); ++ CUDA_CHECK(cudaMemcpyFromSymbol(&count, d_innerq_count, sizeof(int))); ++ int zero = 0; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int))); // stop accumulating ++ innerq_mode = 0; ++ if (count <= 0) { ++ GGML_LOG_WARN("turbo_innerq_finalize: 0 blocks accumulated — nothing written\n"); ++ return; ++ } ++ // opencoti-hook: turboquant InnerQ (M6-S2) — env-tunable equalization clamp. ++ // The original hard [0.5,2.0] cap (max 2x) under-equalizes models with extreme ++ // per-channel K outliers: Qwen2.5-1.5B has ~10/128 K channels needing >=4x (ratio>=4 ++ // ⇒ scale>=2), which the 2x cap saturated, leaving turbo3-K too lossy for softmax ++ // while turbo3-V (smoothing-tolerant) stayed fine. Widen to a 4x default and allow ++ // TURBO_INNERQ_CLAMP=C (scale ∈ [1/C, C]) to sweep without a rebuild (re-read at each ++ // calibration finalize). C clamps total equalization swing; strength shapes its curve. ++ float clamp_hi = 4.0f; ++ { ++ const char * ce = getenv("TURBO_INNERQ_CLAMP"); ++ if (ce && ce[0]) { const float v = (float) atof(ce); if (v >= 1.0f) clamp_hi = v; } ++ } ++ const float clamp_lo = 1.0f / clamp_hi; ++ static float scale_inv[INNERQ_MAX_SLOTS][INNERQ_MAX_CHANNELS]; ++ int out_nembd[INNERQ_MAX_SLOTS], out_hd[INNERQ_MAX_SLOTS]; ++ for (int s = 0; s < innerq_nslots; s++) { ++ const int n_ch = innerq_slot_hd[s]; ++ out_nembd[s] = innerq_slot_nembd[s]; ++ out_hd[s] = n_ch; ++ float rms[INNERQ_MAX_CHANNELS], mean_rms = 0.0f, max_ratio = 0.0f, min_ratio = 1e30f; ++ for (int i = 0; i < n_ch; i++) { rms[i] = sqrtf(sq_accum[s][i] / (float)count); mean_rms += rms[i]; } ++ mean_rms /= (float)n_ch; ++ int n_clamped_hi = 0; ++ for (int i = 0; i < n_ch; i++) { ++ const float ratio = (rms[i] > 1e-10f) ? (mean_rms / rms[i]) : 1.0f; ++ float sc = powf(ratio, innerq_strength); ++ if (sc < clamp_lo) sc = clamp_lo; ++ if (sc > clamp_hi) { sc = clamp_hi; n_clamped_hi++; } ++ scale_inv[s][i] = 1.0f / sc; ++ if (ratio > max_ratio) max_ratio = ratio; ++ if (ratio < min_ratio) min_ratio = ratio; ++ } ++ GGML_LOG_INFO("turbo_innerq_finalize: slot %d (n_embd_gqa=%d head_dim=%d) clamp=[%.3f,%.3f] strength=%.2f → " ++ "%d/%d saturated, max_ratio=%.3f min_ratio=%.3f\n", ++ s, out_nembd[s], n_ch, clamp_lo, clamp_hi, innerq_strength, n_clamped_hi, n_ch, max_ratio, min_ratio); ++ } ++ if (innerq_out_path && innerq_out_path[0]) { ++ const bool ok = turbo_innerq_dump_file_v2(innerq_out_path, innerq_nslots, out_nembd, out_hd, scale_inv); ++ GGML_LOG_INFO("turbo_innerq_finalize: calibrated on %d blocks, %d slot(s) → %s %s\n", ++ count, innerq_nslots, innerq_out_path, ok ? "OK" : "WRITE-FAILED"); ++ } else { ++ GGML_LOG_WARN("turbo_innerq_finalize: calibrated but TURBO_INNERQ_OUT unset — nothing written\n"); ++ } ++} ++ ++// Called before each turbo3 set-rows launch (host). Lazily inits; finalizes when ++// the calibration block target is reached. ++static void turbo_innerq_check_finalize(int group_size) { ++ if (!innerq_initialized) turbo_innerq_init(); ++ if (innerq_mode != 1) return; // only calibrating finalizes ++ if (group_size < 128) { // InnerQ math assumes WHT group == head_dim ++ GGML_LOG_WARN("turbo_innerq_check_finalize: disabled (group_size=%d < 128)\n", group_size); ++ innerq_mode = 0; int zero = 0; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int))); ++ return; ++ } ++ int count = 0; ++ CUDA_CHECK(cudaMemcpyFromSymbol(&count, d_innerq_count, sizeof(int))); ++ if (count >= innerq_target_blocks) turbo_innerq_finalize(group_size); ++} ++// ===== end InnerQ ===== ++ ++template ++__launch_bounds__(128) ++static __global__ void k_set_rows_turbo3( ++ const float * __restrict__ src0, ++ const idx_t * __restrict__ src1, ++ block_turbo3_0 * __restrict__ dst, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne11, const int64_t ne12, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const int innerq_slot) { // InnerQ per-width slot (-1 = off) ++ constexpr int GROUP_SIZE = QK_TURBO3; // 128 ++ const int j = threadIdx.x; // 0..127, element within group ++ ++ // Decode group index → (i_grp, i01, i02, i03). MUST match this tree's ++ // k_set_rows_quant convention (i02 % ne02, i12 = i03 % ne12, i11 = i02 % ne11), ++ // NOT the AtomicBot fork's (which assumes ne02==ne12). ++ const int64_t n_groups_per_row = ne00 / GROUP_SIZE; ++ const int64_t g = blockIdx.x; ++ const int64_t i_grp = g % n_groups_per_row; ++ int64_t tmp = g / n_groups_per_row; ++ const int64_t i01 = tmp % ne01; ++ tmp = tmp / ne01; ++ const int64_t i02 = tmp % ne02; ++ const int64_t i03 = tmp / ne02; ++ ++ const int64_t i12 = i03 % ne12; ++ const int64_t i11 = i02 % ne11; ++ const int64_t i10 = i01; ++ ++ const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); ++ const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03; ++ block_turbo3_0 * dst_row_ptr = (block_turbo3_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3); ++ block_turbo3_0 * blk = dst_row_ptr + i_grp; // blocks_per_group == 1 ++ ++ __shared__ float x[GROUP_SIZE]; ++ x[j] = src_row[i_grp * GROUP_SIZE + j]; ++ __syncthreads(); ++ ++ // opencoti-hook: turboquant InnerQ (M6-S2) — calibrate on RAW values, then equalize. ++ // Calibrate (accumulate K² per channel) and apply (×scale) are mutually exclusive ++ // (calibrate-then-activate flow): calibration runs un-equalized, active runs equalize ++ // from token 0. Both gated on zero-initialized device flags ⇒ OFF mode is inert. ++ // head_dim-aware, PER-WIDTH SLOT: the per-head channel is (i_grp*128 + j) % head_dim ++ // of this layer's width-slot (scale shared across heads of that width). innerq_slot is ++ // resolved host-side from ne00; -1 ⇒ no InnerQ for this width. head_dim==128 ⇒ ch==j ++ // (identity, byte-compatible with the old uniform path). ++ const int innerq_hd = (innerq_slot >= 0) ? d_innerq_slot_hd[innerq_slot] : 0; ++ if (innerq_slot >= 0 && d_innerq_calibrating && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ atomicAdd(&d_innerq_sq_accum[innerq_slot][ch], x[j] * x[j]); ++ if (j == 0) atomicAdd(&d_innerq_count, 1); ++ } ++ if (innerq_slot >= 0 && d_innerq_active && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ x[j] *= d_innerq_scale[innerq_slot][ch]; ++ } ++ __syncthreads(); ++ ++ // Step 2: parallel L2 norm (warp reduce + inter-warp via shared) ++ constexpr int n_warps = GROUP_SIZE / WARP_SIZE; ++ __shared__ float warp_accum[n_warps]; ++ float v2 = x[j] * x[j]; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) v2 += __shfl_xor_sync(0xffffffff, v2, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v2; ++ __syncthreads(); ++ __shared__ float s_norm_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_norm_sq = t; } ++ __syncthreads(); ++ const float grp_norm = sqrtf(s_norm_sq); ++ const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ // Step 3: normalize ++ x[j] *= inv_norm; ++ __syncthreads(); ++ ++ // Step 4: forward WHT (signs1 → butterfly → 1/sqrt128 · signs2) ++ x[j] *= TURBO3_WHT_SIGNS1[j]; ++ __syncthreads(); ++#define WHT_STAGE_SHARED(h) \ ++ if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \ ++ __syncthreads(); ++ WHT_STAGE_SHARED(1) WHT_STAGE_SHARED(2) WHT_STAGE_SHARED(4) ++ WHT_STAGE_SHARED(8) WHT_STAGE_SHARED(16) WHT_STAGE_SHARED(32) WHT_STAGE_SHARED(64) ++#undef WHT_STAGE_SHARED ++ const float inv_sqrt_group = 0.08838834764831845f; // 1/sqrt(128) ++ x[j] = x[j] * inv_sqrt_group * TURBO3_WHT_SIGNS2[j]; ++ __syncthreads(); ++ ++ // Step 5: quantize element j → 3-bit centroid index ++ const uint8_t idx = turbo3_nearest_centroid_3bit(x[j]); ++ ++ // Step 6: warp-cooperative pack (no atomics) ++ const int lane = j % WARP_SIZE; ++ const int elem_in_block = j; // QK_TURBO3 == GROUP_SIZE, single block ++ const int qs_byte_idx = elem_in_block / 4; ++ const uint8_t my_low2 = idx & 0x3; ++ uint8_t qs_byte = 0; ++#pragma unroll ++ for (int k = 0; k < 4; k++) { ++ uint8_t contrib = __shfl_sync(0xffffffff, my_low2, (lane & ~3) + k); ++ qs_byte |= contrib << (k * 2); ++ } ++ if (lane % 4 == 0) blk->qs[qs_byte_idx] = qs_byte; ++ ++ const uint32_t ballot = __ballot_sync(0xffffffff, (idx >> 2) & 1); ++ const int local_signs_byte = lane / 8; ++ const int global_signs_byte = elem_in_block / 8; ++ const uint8_t signs_byte = (uint8_t)((ballot >> (local_signs_byte * 8)) & 0xFF); ++ if (lane % 8 == 0) blk->signs[global_signs_byte] = signs_byte; ++ ++ // Step 7: reconstruction norm (parallel, reuse warp_accum) ++ const float c = TURBO3_CENTROIDS_3BIT[idx]; ++ float rc = c * c; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) rc += __shfl_xor_sync(0xffffffff, rc, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; ++ __syncthreads(); ++ __shared__ float s_recon_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_recon_sq = t; } ++ __syncthreads(); ++ const float recon_norm = sqrtf(s_recon_sq); ++ const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ ++ // Step 8: write corrected norm (one thread per block) ++ if (elem_in_block == 0) blk->norm = __float2half(corrected_norm); ++} ++ ++template ++static void set_rows_cuda_turbo3(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO3 == 0); // 128-block aligned (Gate-2: no tail kernel) ++ ++ cudaStream_t stream = ctx.stream(); ++ const int64_t s01 = nb01 / sizeof(float), s02 = nb02 / sizeof(float), s03 = nb03 / sizeof(float); ++ const int64_t s10 = nb10 / sizeof(idx_t), s11 = nb11 / sizeof(idx_t), s12 = nb12 / sizeof(idx_t); ++ const int64_t n_groups = ne00 / QK_TURBO3; ++ const int64_t ne_total = n_groups * ne01 * ne02 * ne03; ++ if (ne_total <= 0) return; ++ ++ // opencoti-hook: turboquant InnerQ (M6-S2) — lazy init + finalize calibration, then ++ // resolve this layer's per-width slot from ne00 (= n_embd_gqa). -1 ⇒ no InnerQ here. ++ turbo_innerq_check_finalize(QK_TURBO3); ++ const int innerq_slot = (innerq_mode != 0) ? innerq_slot_for((int) ne00) : -1; ++ ++ k_set_rows_turbo3<<<(int)ne_total, QK_TURBO3, 0, stream>>>( ++ src0_d, src1_d, (block_turbo3_0 *)dst->data, ++ ne00, ne01, ne02, ne11, ne12, ++ s01, s02, s03, s10, s11, s12, ++ nb1, nb2, nb3, innerq_slot); ++} ++ ++// ===================== turbo4 (4-bit nibble) write kernel ===================== ++// opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 tier family). Clone of ++// k_set_rows_turbo3: identical InnerQ / L2-norm / forward-WHT preamble (width-independent), ++// differs only at quantize (4-bit centroid), pack (nibble, 2/byte, no QJL signs) and the ++// reconstruction-norm centroid table. block = group = QK_TURBO4 = 128. ++template ++static __global__ void k_set_rows_turbo4( ++ const float * __restrict__ src0, ++ const idx_t * __restrict__ src1, ++ block_turbo4_0 * __restrict__ dst, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne11, const int64_t ne12, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const int innerq_slot) { ++ constexpr int GROUP_SIZE = QK_TURBO4; // 128 ++ const int j = threadIdx.x; ++ ++ const int64_t n_groups_per_row = ne00 / GROUP_SIZE; ++ const int64_t g = blockIdx.x; ++ const int64_t i_grp = g % n_groups_per_row; ++ int64_t tmp = g / n_groups_per_row; ++ const int64_t i01 = tmp % ne01; ++ tmp = tmp / ne01; ++ const int64_t i02 = tmp % ne02; ++ const int64_t i03 = tmp / ne02; ++ const int64_t i12 = i03 % ne12; ++ const int64_t i11 = i02 % ne11; ++ const int64_t i10 = i01; ++ ++ const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); ++ const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03; ++ block_turbo4_0 * dst_row_ptr = (block_turbo4_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3); ++ block_turbo4_0 * blk = dst_row_ptr + i_grp; ++ ++ __shared__ float x[GROUP_SIZE]; ++ x[j] = src_row[i_grp * GROUP_SIZE + j]; ++ __syncthreads(); ++ ++ // InnerQ calibrate/apply (per-width slot) — identical to turbo3. ++ const int innerq_hd = (innerq_slot >= 0) ? d_innerq_slot_hd[innerq_slot] : 0; ++ if (innerq_slot >= 0 && d_innerq_calibrating && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ atomicAdd(&d_innerq_sq_accum[innerq_slot][ch], x[j] * x[j]); ++ if (j == 0) atomicAdd(&d_innerq_count, 1); ++ } ++ if (innerq_slot >= 0 && d_innerq_active && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ x[j] *= d_innerq_scale[innerq_slot][ch]; ++ } ++ __syncthreads(); ++ ++ // L2 norm ++ constexpr int n_warps = GROUP_SIZE / WARP_SIZE; ++ __shared__ float warp_accum[n_warps]; ++ float v2 = x[j] * x[j]; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) v2 += __shfl_xor_sync(0xffffffff, v2, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v2; ++ __syncthreads(); ++ __shared__ float s_norm_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_norm_sq = t; } ++ __syncthreads(); ++ const float grp_norm = sqrtf(s_norm_sq); ++ const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ x[j] *= inv_norm; ++ __syncthreads(); ++ ++ // forward WHT ++ x[j] *= TURBO3_WHT_SIGNS1[j]; ++ __syncthreads(); ++#define WHT_STAGE_T4(h) \ ++ if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \ ++ __syncthreads(); ++ WHT_STAGE_T4(1) WHT_STAGE_T4(2) WHT_STAGE_T4(4) ++ WHT_STAGE_T4(8) WHT_STAGE_T4(16) WHT_STAGE_T4(32) WHT_STAGE_T4(64) ++#undef WHT_STAGE_T4 ++ const float inv_sqrt_group = 0.08838834764831845f; ++ x[j] = x[j] * inv_sqrt_group * TURBO3_WHT_SIGNS2[j]; ++ __syncthreads(); ++ ++ // quantize → 4-bit centroid index ++ const uint8_t idx = turbo4_nearest_centroid_4bit(x[j]); ++ ++ // pack: nibble, 2 elements per byte (low=even, high=odd) — no QJL signs ++ const int lane = j % WARP_SIZE; ++ const int elem_in_block = j; ++ const int qs_byte_idx = elem_in_block / 2; ++ const uint8_t my_nib = idx & 0xF; ++ uint8_t qs_byte = 0; ++#pragma unroll ++ for (int k = 0; k < 2; k++) { ++ uint8_t contrib = __shfl_sync(0xffffffff, my_nib, (lane & ~1) + k); ++ qs_byte |= contrib << (k * 4); ++ } ++ if (lane % 2 == 0) blk->qs[qs_byte_idx] = qs_byte; ++ ++ // reconstruction norm ++ const float c = TURBO4_CENTROIDS_4BIT[idx]; ++ float rc = c * c; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) rc += __shfl_xor_sync(0xffffffff, rc, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; ++ __syncthreads(); ++ __shared__ float s_recon_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_recon_sq = t; } ++ __syncthreads(); ++ const float recon_norm = sqrtf(s_recon_sq); ++ const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ ++ if (elem_in_block == 0) { blk->norm = __float2half(corrected_norm); blk->rnorm = __float2half(0.0f); } ++} ++ ++template ++static void set_rows_cuda_turbo4(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO4 == 0); ++ ++ cudaStream_t stream = ctx.stream(); ++ const int64_t s01 = nb01 / sizeof(float), s02 = nb02 / sizeof(float), s03 = nb03 / sizeof(float); ++ const int64_t s10 = nb10 / sizeof(idx_t), s11 = nb11 / sizeof(idx_t), s12 = nb12 / sizeof(idx_t); ++ const int64_t n_groups = ne00 / QK_TURBO4; ++ const int64_t ne_total = n_groups * ne01 * ne02 * ne03; ++ if (ne_total <= 0) return; ++ ++ turbo_innerq_check_finalize(QK_TURBO4); ++ const int innerq_slot = (innerq_mode != 0) ? innerq_slot_for((int) ne00) : -1; ++ ++ k_set_rows_turbo4<<<(int)ne_total, QK_TURBO4, 0, stream>>>( ++ src0_d, src1_d, (block_turbo4_0 *)dst->data, ++ ne00, ne01, ne02, ne11, ne12, ++ s01, s02, s03, s10, s11, s12, ++ nb1, nb2, nb3, innerq_slot); ++} ++ ++// ===================== turbo2 (2-bit, 4/byte) write kernel ===================== ++// Clone of turbo3 minus QJL signs: 2-bit centroid + 4-per-byte pack (idx already 2-bit). ++template ++static __global__ void k_set_rows_turbo2( ++ const float * __restrict__ src0, ++ const idx_t * __restrict__ src1, ++ block_turbo2_0 * __restrict__ dst, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne11, const int64_t ne12, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const int innerq_slot) { ++ constexpr int GROUP_SIZE = QK_TURBO2; // 128 ++ const int j = threadIdx.x; ++ ++ const int64_t n_groups_per_row = ne00 / GROUP_SIZE; ++ const int64_t g = blockIdx.x; ++ const int64_t i_grp = g % n_groups_per_row; ++ int64_t tmp = g / n_groups_per_row; ++ const int64_t i01 = tmp % ne01; ++ tmp = tmp / ne01; ++ const int64_t i02 = tmp % ne02; ++ const int64_t i03 = tmp / ne02; ++ const int64_t i12 = i03 % ne12; ++ const int64_t i11 = i02 % ne11; ++ const int64_t i10 = i01; ++ ++ const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); ++ const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03; ++ block_turbo2_0 * dst_row_ptr = (block_turbo2_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3); ++ block_turbo2_0 * blk = dst_row_ptr + i_grp; ++ ++ __shared__ float x[GROUP_SIZE]; ++ x[j] = src_row[i_grp * GROUP_SIZE + j]; ++ __syncthreads(); ++ ++ const int innerq_hd = (innerq_slot >= 0) ? d_innerq_slot_hd[innerq_slot] : 0; ++ if (innerq_slot >= 0 && d_innerq_calibrating && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ atomicAdd(&d_innerq_sq_accum[innerq_slot][ch], x[j] * x[j]); ++ if (j == 0) atomicAdd(&d_innerq_count, 1); ++ } ++ if (innerq_slot >= 0 && d_innerq_active && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ x[j] *= d_innerq_scale[innerq_slot][ch]; ++ } ++ __syncthreads(); ++ ++ constexpr int n_warps = GROUP_SIZE / WARP_SIZE; ++ __shared__ float warp_accum[n_warps]; ++ float v2 = x[j] * x[j]; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) v2 += __shfl_xor_sync(0xffffffff, v2, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v2; ++ __syncthreads(); ++ __shared__ float s_norm_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_norm_sq = t; } ++ __syncthreads(); ++ const float grp_norm = sqrtf(s_norm_sq); ++ const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ x[j] *= inv_norm; ++ __syncthreads(); ++ ++ x[j] *= TURBO3_WHT_SIGNS1[j]; ++ __syncthreads(); ++#define WHT_STAGE_T2(h) \ ++ if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \ ++ __syncthreads(); ++ WHT_STAGE_T2(1) WHT_STAGE_T2(2) WHT_STAGE_T2(4) ++ WHT_STAGE_T2(8) WHT_STAGE_T2(16) WHT_STAGE_T2(32) WHT_STAGE_T2(64) ++#undef WHT_STAGE_T2 ++ const float inv_sqrt_group = 0.08838834764831845f; ++ x[j] = x[j] * inv_sqrt_group * TURBO3_WHT_SIGNS2[j]; ++ __syncthreads(); ++ ++ const uint8_t idx = turbo2_nearest_centroid_2bit(x[j]); ++ ++ const int lane = j % WARP_SIZE; ++ const int elem_in_block = j; ++ const int qs_byte_idx = elem_in_block / 4; ++ const uint8_t my_2bit = idx & 0x3; ++ uint8_t qs_byte = 0; ++#pragma unroll ++ for (int k = 0; k < 4; k++) { ++ uint8_t contrib = __shfl_sync(0xffffffff, my_2bit, (lane & ~3) + k); ++ qs_byte |= contrib << (k * 2); ++ } ++ if (lane % 4 == 0) blk->qs[qs_byte_idx] = qs_byte; ++ ++ const float c = TURBO2_CENTROIDS_2BIT[idx]; ++ float rc = c * c; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) rc += __shfl_xor_sync(0xffffffff, rc, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; ++ __syncthreads(); ++ __shared__ float s_recon_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_recon_sq = t; } ++ __syncthreads(); ++ const float recon_norm = sqrtf(s_recon_sq); ++ const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ ++ if (elem_in_block == 0) blk->norm = __float2half(corrected_norm); ++} ++ ++template ++static void set_rows_cuda_turbo2(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO2 == 0); ++ ++ cudaStream_t stream = ctx.stream(); ++ const int64_t s01 = nb01 / sizeof(float), s02 = nb02 / sizeof(float), s03 = nb03 / sizeof(float); ++ const int64_t s10 = nb10 / sizeof(idx_t), s11 = nb11 / sizeof(idx_t), s12 = nb12 / sizeof(idx_t); ++ const int64_t n_groups = ne00 / QK_TURBO2; ++ const int64_t ne_total = n_groups * ne01 * ne02 * ne03; ++ if (ne_total <= 0) return; ++ ++ turbo_innerq_check_finalize(QK_TURBO2); ++ const int innerq_slot = (innerq_mode != 0) ? innerq_slot_for((int) ne00) : -1; ++ ++ k_set_rows_turbo2<<<(int)ne_total, QK_TURBO2, 0, stream>>>( ++ src0_d, src1_d, (block_turbo2_0 *)dst->data, ++ ne00, ne01, ne02, ne11, ne12, ++ s01, s02, s03, s10, s11, s12, ++ nb1, nb2, nb3, innerq_slot); ++} ++ ++// ===================== turbo8 (8-bit uniform int8) write kernel ===================== ++// Clone of turbo3 preamble; quantize = uniform int8 (q=clamp(round(x*TURBO8_SCALE),-127,127)), ++// pack = 1 byte/element (no warp packing), recon centroid = q/TURBO8_SCALE. ++template ++static __global__ void k_set_rows_turbo8( ++ const float * __restrict__ src0, ++ const idx_t * __restrict__ src1, ++ block_turbo8_0 * __restrict__ dst, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne11, const int64_t ne12, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const int innerq_slot) { ++ constexpr int GROUP_SIZE = QK_TURBO8; // 128 ++ const int j = threadIdx.x; ++ ++ const int64_t n_groups_per_row = ne00 / GROUP_SIZE; ++ const int64_t g = blockIdx.x; ++ const int64_t i_grp = g % n_groups_per_row; ++ int64_t tmp = g / n_groups_per_row; ++ const int64_t i01 = tmp % ne01; ++ tmp = tmp / ne01; ++ const int64_t i02 = tmp % ne02; ++ const int64_t i03 = tmp / ne02; ++ const int64_t i12 = i03 % ne12; ++ const int64_t i11 = i02 % ne11; ++ const int64_t i10 = i01; ++ ++ const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); ++ const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03; ++ block_turbo8_0 * dst_row_ptr = (block_turbo8_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3); ++ block_turbo8_0 * blk = dst_row_ptr + i_grp; ++ ++ __shared__ float x[GROUP_SIZE]; ++ x[j] = src_row[i_grp * GROUP_SIZE + j]; ++ __syncthreads(); ++ ++ const int innerq_hd = (innerq_slot >= 0) ? d_innerq_slot_hd[innerq_slot] : 0; ++ if (innerq_slot >= 0 && d_innerq_calibrating && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ atomicAdd(&d_innerq_sq_accum[innerq_slot][ch], x[j] * x[j]); ++ if (j == 0) atomicAdd(&d_innerq_count, 1); ++ } ++ if (innerq_slot >= 0 && d_innerq_active && innerq_hd > 0) { ++ const int ch = (int)((i_grp * GROUP_SIZE + j) % innerq_hd); ++ x[j] *= d_innerq_scale[innerq_slot][ch]; ++ } ++ __syncthreads(); ++ ++ constexpr int n_warps = GROUP_SIZE / WARP_SIZE; ++ __shared__ float warp_accum[n_warps]; ++ float v2 = x[j] * x[j]; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) v2 += __shfl_xor_sync(0xffffffff, v2, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v2; ++ __syncthreads(); ++ __shared__ float s_norm_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_norm_sq = t; } ++ __syncthreads(); ++ const float grp_norm = sqrtf(s_norm_sq); ++ const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ x[j] *= inv_norm; ++ __syncthreads(); ++ ++ x[j] *= TURBO3_WHT_SIGNS1[j]; ++ __syncthreads(); ++#define WHT_STAGE_T8(h) \ ++ if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \ ++ __syncthreads(); ++ WHT_STAGE_T8(1) WHT_STAGE_T8(2) WHT_STAGE_T8(4) ++ WHT_STAGE_T8(8) WHT_STAGE_T8(16) WHT_STAGE_T8(32) WHT_STAGE_T8(64) ++#undef WHT_STAGE_T8 ++ const float inv_sqrt_group = 0.08838834764831845f; ++ x[j] = x[j] * inv_sqrt_group * TURBO3_WHT_SIGNS2[j]; ++ __syncthreads(); ++ ++ // quantize → signed int8 (saturating), 1 byte/element ++ int q = __float2int_rn(x[j] * TURBO8_SCALE); ++ if (q > 127) q = 127; ++ if (q < -127) q = -127; ++ blk->qs[j] = (int8_t) q; ++ ++ const float c = (float) q / TURBO8_SCALE; ++ float rc = c * c; ++ for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) rc += __shfl_xor_sync(0xffffffff, rc, offset); ++ if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; ++ __syncthreads(); ++ __shared__ float s_recon_sq; ++ if (j == 0) { float t = 0.0f; for (int w = 0; w < n_warps; w++) t += warp_accum[w]; s_recon_sq = t; } ++ __syncthreads(); ++ const float recon_norm = sqrtf(s_recon_sq); ++ const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ ++ if (j == 0) blk->norm = __float2half(corrected_norm); ++} ++ ++template ++static void set_rows_cuda_turbo8(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO8 == 0); ++ ++ cudaStream_t stream = ctx.stream(); ++ const int64_t s01 = nb01 / sizeof(float), s02 = nb02 / sizeof(float), s03 = nb03 / sizeof(float); ++ const int64_t s10 = nb10 / sizeof(idx_t), s11 = nb11 / sizeof(idx_t), s12 = nb12 / sizeof(idx_t); ++ const int64_t n_groups = ne00 / QK_TURBO8; ++ const int64_t ne_total = n_groups * ne01 * ne02 * ne03; ++ if (ne_total <= 0) return; ++ ++ turbo_innerq_check_finalize(QK_TURBO8); ++ const int innerq_slot = (innerq_mode != 0) ? innerq_slot_for((int) ne00) : -1; ++ ++ k_set_rows_turbo8<<<(int)ne_total, QK_TURBO8, 0, stream>>>( ++ src0_d, src1_d, (block_turbo8_0 *)dst->data, ++ ne00, ne01, ne02, ne11, ne12, ++ s01, s02, s03, s10, s11, s12, ++ nb1, nb2, nb3, innerq_slot); ++} ++ + template + static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const src_t * src0_d = (const src_t *)src0->data; +@@ -314,6 +1114,18 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s + nb1, nb2, nb3, + stream + ); ++ } else if (dst->type == GGML_TYPE_TURBO3_0) { ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2 write path) ++ set_rows_cuda_turbo3(ctx, src0, src1, dst); ++ } else if (dst->type == GGML_TYPE_TURBO4_0) { ++ // opencoti-hook: turboquant tier family — see docs/features/poly_kv.md (M6-S2) ++ set_rows_cuda_turbo4(ctx, src0, src1, dst); ++ } else if (dst->type == GGML_TYPE_TURBO2_0) { ++ // opencoti-hook: turboquant tier family — see docs/features/poly_kv.md (M6-S2) ++ set_rows_cuda_turbo2(ctx, src0, src1, dst); ++ } else if (dst->type == GGML_TYPE_TURBO8_0) { ++ // opencoti-hook: turboquant tier family — see docs/features/poly_kv.md (M6-S2) ++ set_rows_cuda_turbo8(ctx, src0, src1, dst); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu +@@ -0,0 +1,8 @@ ++// This file has been autogenerated by generate_cu_files.py, do not edit manually. ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu +@@ -0,0 +1,8 @@ ++// This file has been autogenerated by generate_cu_files.py, do not edit manually. ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu +@@ -0,0 +1,8 @@ ++// This file has been autogenerated by generate_cu_files.py, do not edit manually. ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +@@ -74,6 +74,12 @@ for type_k in TYPES_KV: + with open(f"fattn-vec-instance-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f: + f.write(SOURCE_FATTN_VEC.format(type_k=type_k, type_v=type_v)) + ++# opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec instances ++# (K==V same tier only; the in-register readers live in fattn-turbo.cuh). ++for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]: ++ with open(f"fattn-vec-instance-{get_short_name(type_t)}-{get_short_name(type_t)}.cu", "w") as f: ++ f.write(SOURCE_FATTN_VEC.format(type_k=type_t, type_v=type_t)) ++ + for ncols in [8, 16, 32, 64]: + for ncols2 in [1, 2, 4, 8, 16, 32]: + if ncols2 > ncols: +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu +@@ -0,0 +1,133 @@ ++// opencoti-hook: turboquant perf-lift (M6-S2 Level A) — see docs/features/poly_kv.md ++// ++// GPU-resident dequant-on-lift kernels for TurboQuant KV (turbo2/3/4/8 → f16). One CUDA threadblock ++// (128 threads) per 128-element WHT group: load codebook/int8 → centroid×norm into shared mem → ++// inverse WHT computed ONCE cooperatively (not per output pair) → ×scale_inv[ch] (InnerQ, true ++// space) → strided f16 write. Numerically mirrors the CPU dequant dequantize_row_turboN_0 ++// (ggml-turbo-quant.c): same inverse-WHT order (signs2 → butterfly → 1/√128·signs1), same per-width ++// channel fold (global_index % head_dim), same InnerQ un-equalize. Accept = logit-equivalence. ++ ++#include "common.cuh" ++#include "turbo-dequant.cuh" // TURBO{3}_WHT_SIGNS1/2 + TURBO{2,3,4}_CENTROIDS tables (this TU's copy) ++#include "turbo-cpy.cuh" ++ ++// Cooperative inverse WHT over shared buf[128] (already holding centroid×norm in rotated space), ++// then InnerQ ×scale_inv and a strided f16 write of element e0+t. All 128 threads of the block ++// participate; callers guarantee e0+t < ne (no divergent early-return before the syncs). ++static __device__ __forceinline__ void turbo_wht_inv_write( ++ float * buf, int t, int64_t e0, char * cdst, ++ int64_t ne10, int64_t ne11, int64_t ne12, ++ int64_t nb10, int64_t nb11, int64_t nb12, int64_t nb13, ++ const float * scale_inv, int head_dim) { ++ __syncthreads(); ++ buf[t] *= TURBO3_WHT_SIGNS2[t]; ++ for (int h = 1; h < 128; h <<= 1) { ++ __syncthreads(); ++ if ((t & h) == 0) { ++ const float a = buf[t], b = buf[t + h]; ++ buf[t] = a + b; ++ buf[t + h] = a - b; ++ } ++ } ++ __syncthreads(); ++ float val = buf[t] * 0.08838834764831845f * TURBO3_WHT_SIGNS1[t]; ++ if (scale_inv) { ++ val *= scale_inv[(int) ((e0 + t) % head_dim)]; ++ } ++ const int64_t e = e0 + t; ++ const int64_t i13 = e / (ne10 * ne11 * ne12); ++ const int64_t i12 = (e - i13*ne10*ne11*ne12) / (ne10*ne11); ++ const int64_t i11 = (e - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10; ++ const int64_t i10 = e - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; ++ const int64_t dst_off = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13; ++ *(half *) (cdst + dst_off) = __float2half(val); ++} ++ ++// Compute the (shared) turbo block pointer for group e0 from the strided src layout. ++#define TURBO_SRC_BLOCK(block_t) \ ++ const int64_t i03 = e0 / (ne00 * ne01 * ne02); \ ++ const int64_t i02 = (e0 - i03*ne00*ne01*ne02) / (ne00*ne01); \ ++ const int64_t i01 = (e0 - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00; \ ++ const int64_t i00 = e0 - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00; \ ++ const int64_t x_off = (i00 / 128)*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ ++ const block_t * blk = (const block_t *) (cx + x_off) ++ ++#define TURBO_CPY_PARAMS \ ++ const char * cx, char * cdst, int64_t ne, \ ++ int64_t ne00, int64_t ne01, int64_t ne02, \ ++ int64_t nb00, int64_t nb01, int64_t nb02, int64_t nb03, \ ++ int64_t ne10, int64_t ne11, int64_t ne12, \ ++ int64_t nb10, int64_t nb11, int64_t nb12, int64_t nb13, \ ++ const float * scale_inv, int head_dim ++ ++static __global__ void cpy_turbo3_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo3_0); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ const uint8_t low2 = (blk->qs[t / 4] >> ((t % 4) * 2)) & 0x3; ++ const uint8_t hi1 = (blk->signs[t / 8] >> (t % 8)) & 0x1; ++ buf[t] = TURBO3_CENTROIDS_3BIT[(uint8_t)(low2 | (hi1 << 2))] * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, scale_inv, head_dim); ++} ++ ++static __global__ void cpy_turbo4_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo4_0); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ const uint8_t idx = (blk->qs[t / 2] >> ((t % 2) * 4)) & 0xF; ++ buf[t] = TURBO4_CENTROIDS_4BIT[idx] * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, scale_inv, head_dim); ++} ++ ++static __global__ void cpy_turbo2_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo2_0); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ const uint8_t idx = (blk->qs[t / 4] >> ((t % 4) * 2)) & 0x3; ++ buf[t] = TURBO2_CENTROIDS_2BIT[idx] * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, scale_inv, head_dim); ++} ++ ++static __global__ void cpy_turbo8_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo8_0); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ buf[t] = (float) blk->qs[t] * (1.0f / TURBO8_SCALE) * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, scale_inv, head_dim); ++} ++ ++void ggml_cpy_turbo_f16_cuda( ++ enum ggml_type src_type, const char * cx, char * cdst, int64_t ne, ++ int64_t ne00, int64_t ne01, int64_t ne02, ++ int64_t nb00, int64_t nb01, int64_t nb02, int64_t nb03, ++ int64_t ne10, int64_t ne11, int64_t ne12, ++ int64_t nb10, int64_t nb11, int64_t nb12, int64_t nb13, ++ const float * d_scale_inv, int head_dim, cudaStream_t stream) { ++ GGML_ASSERT(ne % 128 == 0); ++ const int64_t num_groups = ne / 128; ++ GGML_ASSERT(num_groups < UINT_MAX); ++ const dim3 grid((unsigned) num_groups); ++ #define LAUNCH(K) K<<>>( \ ++ cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, \ ++ ne10, ne11, ne12, nb10, nb11, nb12, nb13, d_scale_inv, head_dim) ++ switch (src_type) { ++ case GGML_TYPE_TURBO3_0: LAUNCH(cpy_turbo3_f16); break; ++ case GGML_TYPE_TURBO4_0: LAUNCH(cpy_turbo4_f16); break; ++ case GGML_TYPE_TURBO2_0: LAUNCH(cpy_turbo2_f16); break; ++ case GGML_TYPE_TURBO8_0: LAUNCH(cpy_turbo8_f16); break; ++ default: GGML_ABORT("ggml_cpy_turbo_f16_cuda: non-turbo src_type %d\n", (int) src_type); ++ } ++ #undef LAUNCH ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cuh +@@ -0,0 +1,25 @@ ++// opencoti-hook: turboquant perf-lift (M6-S2 Level A) — see docs/features/poly_kv.md ++// ++// GPU-resident dequant-on-lift for TurboQuant KV. Replaces the host round-trip (the scheduler used ++// to spill ggml_cast(turboN→f16) to the CPU backend because CUDA's CPY supports_op rejected turbo) ++// with a single block-cooperative kernel: one CUDA threadblock per 128-element WHT group computes ++// the inverse WHT ONCE (vs the per-output-pair recompute in turbo-dequant.cuh's convert path) and ++// applies InnerQ ×scale_inv in true-channel space. Output is f16 in VRAM; stock flash-attention ++// then runs unchanged. Numerically reproduces the CPU dequant (dequantize_row_turboN_0); gated by ++// logit-equivalence, never byte-equality. ++#pragma once ++ ++#include "ggml.h" ++#include ++ ++// Dispatch a turboN→f16 dequant-cpy on the GPU. src_type ∈ {TURBO2_0,TURBO3_0,TURBO4_0,TURBO8_0}. ++// d_scale_inv: device pointer to the active InnerQ slot's scale_inv[head_dim] (from ++// turbo_innerq_active_device_scale), or nullptr ⇒ no un-equalize (identity). ne must be a multiple ++// of 128 (the WHT group / turbo block size). ++void ggml_cpy_turbo_f16_cuda( ++ enum ggml_type src_type, const char * cx, char * cdst, int64_t ne, ++ int64_t ne00, int64_t ne01, int64_t ne02, ++ int64_t nb00, int64_t nb01, int64_t nb02, int64_t nb03, ++ int64_t ne10, int64_t ne11, int64_t ne12, ++ int64_t nb10, int64_t nb11, int64_t nb12, int64_t nb13, ++ const float * d_scale_inv, int head_dim, cudaStream_t stream); +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-dequant.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-dequant.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-dequant.cuh +@@ -0,0 +1,213 @@ ++// opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2) ++// ++// Minimal, InnerQ-FREE CUDA dequant for TurboQuant turbo3 KV, used by the ++// Gate-2 "dequant-then-standard-FA" fallback. Folds the INVERSE WHT into the ++// block dequant so the emitted f16 is in TRUE (un-rotated) space — the ++// standard f16 flash-attention then computes q·k directly with NO q-rotation ++// hook. (The fork's hot fused kernels instead store rotated K and rotate q; ++// that path + InnerQ Q/V-equalization is Gate-3.) ++// ++// turbo3 block = 128 values = exactly one WHT group, so the inverse WHT is a ++// self-contained per-block 128-pt butterfly (cold path: recomputed per element ++// pair, like the fork's dequantize_tq3_1s). Constants copied verbatim from the ++// AtomicBot fork ggml-cuda/turbo-quant.cuh; inverse-WHT order mirrors the ++// cosine-0.983-validated turbo_cpu_fwht_inverse (signs2 → butterfly → 1/√128·signs1). ++#pragma once ++ ++#include "common.cuh" ++ ++#define QK_TURBO3_INV 128 ++ ++static __constant__ float TURBO3_CENTROIDS_3BIT[8] = { ++ -0.190685f, -0.117832f, -0.065717f, -0.021460f, ++ 0.021460f, 0.065717f, 0.117832f, 0.190685f ++}; ++ ++static __constant__ float TURBO3_WHT_SIGNS1[128] = { ++ -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, ++ -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, ++ 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, ++ -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, ++ -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f ++}; ++ ++static __constant__ float TURBO3_WHT_SIGNS2[128] = { ++ 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, ++ 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, ++ 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, ++ -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f ++}; ++ ++// ---- Forward-quantize side (used by set-rows.cu k_set_rows_turbo3, M6-S2 Gate-2 write path) ---- ++static __constant__ float TURBO3_MID_3BIT[7] = { ++ -0.154259f, -0.091775f, -0.043589f, 0.0f, ++ 0.043589f, 0.091775f, 0.154259f ++}; ++ ++static __device__ __forceinline__ uint8_t turbo3_nearest_centroid_3bit(float val) { ++ if (val < TURBO3_MID_3BIT[0]) return 0; ++ else if (val < TURBO3_MID_3BIT[1]) return 1; ++ else if (val < TURBO3_MID_3BIT[2]) return 2; ++ else if (val < TURBO3_MID_3BIT[3]) return 3; ++ else if (val < TURBO3_MID_3BIT[4]) return 4; ++ else if (val < TURBO3_MID_3BIT[5]) return 5; ++ else if (val < TURBO3_MID_3BIT[6]) return 6; ++ else return 7; ++} ++ ++// Dequant one turbo3 block (128 vals) into TRUE space: centroid lookup → inverse WHT. ++// Cold path — the full 128-pt butterfly is recomputed per (iqs) pair, matching the ++// fork's dequantize_tq3_1s cold-dequant pattern. Correctness over speed (Gate-3 fuses). ++static __device__ __forceinline__ void dequantize_turbo3_0_inv( ++ const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo3_0 * x = (const block_turbo3_0 *) vx; ++ const float norm = __half2float(x[ib].norm); ++ ++ float buf[128]; ++ // centroid lookup (rotated space) ++ for (int j = 0; j < 128; j++) { ++ const uint8_t low2 = (x[ib].qs[j / 4] >> ((j % 4) * 2)) & 0x3; ++ const uint8_t hi1 = (x[ib].signs[j / 8] >> (j % 8)) & 0x1; ++ const uint8_t idx = (uint8_t) (low2 | (hi1 << 2)); ++ buf[j] = TURBO3_CENTROIDS_3BIT[idx] * norm; ++ } ++ // inverse WHT: signs2 → butterfly → (1/sqrt128)·signs1 (validated order) ++ for (int j = 0; j < 128; j++) buf[j] *= TURBO3_WHT_SIGNS2[j]; ++ for (int h = 1; h < 128; h <<= 1) { ++ for (int i = 0; i < 128; i += h << 1) { ++ for (int j = i; j < i + h; j++) { ++ const float a = buf[j], b = buf[j + h]; ++ buf[j] = a + b; buf[j + h] = a - b; ++ } ++ } ++ } ++ const float inv_sqrt128 = 0.08838834764831845f; ++ for (int j = 0; j < 128; j++) buf[j] *= inv_sqrt128 * TURBO3_WHT_SIGNS1[j]; ++ ++ v.x = buf[iqs + 0]; ++ v.y = buf[iqs + 1]; ++} ++ ++// ===================== turbo4 (4-bit nibble PolarQuant) ===================== ++// Same WHT group (128) ⇒ reuse TURBO3_WHT_SIGNS1/2 (width-independent). 16 MSE-optimal ++// centroids for N(0,1/128), nibble-packed, no QJL signs. Centroids + midpoints mirror the ++// CPU nearest_centroid_4bit / CENTROIDS_4BIT so GPU-write ↔ CPU-dequant indices agree. ++static __constant__ float TURBO4_CENTROIDS_4BIT[16] = { ++ -0.173926f, -0.117195f, -0.089527f, -0.068756f, ++ -0.051262f, -0.035597f, -0.020989f, -0.006938f, ++ 0.006938f, 0.020989f, 0.035597f, 0.051262f, ++ 0.068756f, 0.089527f, 0.117195f, 0.173926f ++}; ++static __constant__ float TURBO4_MID_4BIT[15] = { ++ -0.145560f, -0.103361f, -0.079142f, -0.060009f, ++ -0.043430f, -0.028293f, -0.013963f, 0.0f, ++ 0.013963f, 0.028293f, 0.043430f, 0.060009f, ++ 0.079142f, 0.103361f, 0.145560f ++}; ++static __device__ __forceinline__ uint8_t turbo4_nearest_centroid_4bit(float val) { ++ if (val < TURBO4_MID_4BIT[0]) return 0; ++ else if (val < TURBO4_MID_4BIT[1]) return 1; ++ else if (val < TURBO4_MID_4BIT[2]) return 2; ++ else if (val < TURBO4_MID_4BIT[3]) return 3; ++ else if (val < TURBO4_MID_4BIT[4]) return 4; ++ else if (val < TURBO4_MID_4BIT[5]) return 5; ++ else if (val < TURBO4_MID_4BIT[6]) return 6; ++ else if (val < TURBO4_MID_4BIT[7]) return 7; ++ else if (val < TURBO4_MID_4BIT[8]) return 8; ++ else if (val < TURBO4_MID_4BIT[9]) return 9; ++ else if (val < TURBO4_MID_4BIT[10]) return 10; ++ else if (val < TURBO4_MID_4BIT[11]) return 11; ++ else if (val < TURBO4_MID_4BIT[12]) return 12; ++ else if (val < TURBO4_MID_4BIT[13]) return 13; ++ else if (val < TURBO4_MID_4BIT[14]) return 14; ++ else return 15; ++} ++static __device__ __forceinline__ void dequantize_turbo4_0_inv( ++ const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo4_0 * x = (const block_turbo4_0 *) vx; ++ const float norm = __half2float(x[ib].norm); ++ float buf[128]; ++ for (int j = 0; j < 128; j++) { ++ const uint8_t idx = (x[ib].qs[j / 2] >> ((j % 2) * 4)) & 0xF; ++ buf[j] = TURBO4_CENTROIDS_4BIT[idx] * norm; ++ } ++ for (int j = 0; j < 128; j++) buf[j] *= TURBO3_WHT_SIGNS2[j]; ++ for (int h = 1; h < 128; h <<= 1) { ++ for (int i = 0; i < 128; i += h << 1) { ++ for (int j = i; j < i + h; j++) { ++ const float a = buf[j], b = buf[j + h]; ++ buf[j] = a + b; buf[j + h] = a - b; ++ } ++ } ++ } ++ const float inv_sqrt128 = 0.08838834764831845f; ++ for (int j = 0; j < 128; j++) buf[j] *= inv_sqrt128 * TURBO3_WHT_SIGNS1[j]; ++ v.x = buf[iqs + 0]; ++ v.y = buf[iqs + 1]; ++} ++ ++// ===================== turbo8 (8-bit uniform int8) ===================== ++// q/TURBO8_SCALE → inverse WHT (no codebook). Reuses TURBO3_WHT_SIGNS1/2 (width-independent). ++static __device__ __forceinline__ void dequantize_turbo8_0_inv( ++ const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo8_0 * x = (const block_turbo8_0 *) vx; ++ const float norm = __half2float(x[ib].norm); ++ const float inv_scale = 1.0f / TURBO8_SCALE; ++ float buf[128]; ++ for (int j = 0; j < 128; j++) buf[j] = (float) x[ib].qs[j] * inv_scale * norm; ++ for (int j = 0; j < 128; j++) buf[j] *= TURBO3_WHT_SIGNS2[j]; ++ for (int h = 1; h < 128; h <<= 1) { ++ for (int i = 0; i < 128; i += h << 1) { ++ for (int j = i; j < i + h; j++) { ++ const float a = buf[j], b = buf[j + h]; ++ buf[j] = a + b; buf[j + h] = a - b; ++ } ++ } ++ } ++ const float inv_sqrt128 = 0.08838834764831845f; ++ for (int j = 0; j < 128; j++) buf[j] *= inv_sqrt128 * TURBO3_WHT_SIGNS1[j]; ++ v.x = buf[iqs + 0]; ++ v.y = buf[iqs + 1]; ++} ++ ++// ===================== turbo2 (2-bit PolarQuant, no QJL) ===================== ++static __constant__ float TURBO2_CENTROIDS_2BIT[4] = { ++ -0.133462f, -0.039994f, 0.039994f, 0.133462f ++}; ++static __constant__ float TURBO2_MID_2BIT[3] = { -0.086728f, 0.0f, 0.086728f }; ++static __device__ __forceinline__ uint8_t turbo2_nearest_centroid_2bit(float val) { ++ if (val < TURBO2_MID_2BIT[0]) return 0; ++ else if (val < TURBO2_MID_2BIT[1]) return 1; ++ else if (val < TURBO2_MID_2BIT[2]) return 2; ++ else return 3; ++} ++static __device__ __forceinline__ void dequantize_turbo2_0_inv( ++ const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo2_0 * x = (const block_turbo2_0 *) vx; ++ const float norm = __half2float(x[ib].norm); ++ float buf[128]; ++ for (int j = 0; j < 128; j++) { ++ const uint8_t idx = (x[ib].qs[j / 4] >> ((j % 4) * 2)) & 0x3; ++ buf[j] = TURBO2_CENTROIDS_2BIT[idx] * norm; ++ } ++ for (int j = 0; j < 128; j++) buf[j] *= TURBO3_WHT_SIGNS2[j]; ++ for (int h = 1; h < 128; h <<= 1) { ++ for (int i = 0; i < 128; i += h << 1) { ++ for (int j = i; j < i + h; j++) { ++ const float a = buf[j], b = buf[j + h]; ++ buf[j] = a + b; buf[j + h] = a - b; ++ } ++ } ++ } ++ const float inv_sqrt128 = 0.08838834764831845f; ++ for (int j = 0; j < 128; j++) buf[j] *= inv_sqrt128 * TURBO3_WHT_SIGNS1[j]; ++ v.x = buf[iqs + 0]; ++ v.y = buf[iqs + 1]; ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-innerq-dev.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-innerq-dev.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-innerq-dev.cuh +@@ -0,0 +1,21 @@ ++// opencoti-hook: turboquant perf-lift (M6-S2 Level A) — see docs/features/poly_kv.md ++// ++// Cross-TU bridge for the GPU-resident dequant-on-lift. The InnerQ per-width scale_inv lives as ++// TU-local `static __device__` symbols in set-rows.cu (d_innerq_scale_inv) and is therefore not ++// reachable from another translation unit. This accessor — implemented in set-rows.cu, declared ++// here, called by turbo-cpy.cu — exposes the device pointer to the *active* slot's scale_inv row ++// plus its head_dim, selected by the cast's row width (ne00 = n_embd_gqa). It mirrors the CPU ++// dequant's slot selection (turbo_innerq_cpu_slot_for in ggml-turbo-quant.c) so GPU lift and CPU ++// lift apply identical InnerQ un-equalization. ++// ++// Returns false when InnerQ is inactive (default-off, calibrating, or no slot matches this width) — ++// the caller then performs the inverse-WHT dequant with NO ×scale_inv (identity), exactly as the ++// CPU path does when its slot lookup misses. ++#pragma once ++ ++#include ++ ++// d_scale_inv_out: device pointer to scale_inv[head_dim] for the matched width (valid while the ++// process lives; the symbol is static-storage device memory). head_dim_out: that slot's head_dim ++// (the channel fold modulus). Both written only when the function returns true. ++bool turbo_innerq_active_device_scale(int64_t ne00, const float ** d_scale_inv_out, int * head_dim_out); +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-innerq.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-innerq.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-innerq.cuh +@@ -0,0 +1,114 @@ ++// opencoti-hook: turboquant InnerQ (M6-S2) — see docs/features/poly_kv.md ++// ++// TurboQuant InnerQ per-channel equalization — calibration artifact I/O. ++// ++// InnerQ equalizes per-channel K variance BEFORE the WHT rotation so the shared ++// 3-bit centroid codebook fits each channel: forward does x[j] *= scale[j], the ++// dequant undoes it with x[j] *= scale_inv[j] in true-channel space. This is the ++// missing correctness ingredient that makes turbo3 usable as a K-quant (turbo3-K ++// at strength-0 is only ~0.983 cosine — too lossy for softmax attention scores). ++// ++// opencoti uses a CALIBRATE-then-ACTIVATE-FROM-TOKEN-0 flow so the whole KV cache ++// is uniformly equalized (no mixed early/late tokens). Because the turbo3 write is ++// GPU-only (this DSO) but our active dequant is the CPU one (host binary), the ++// scale_inv[128] vector crosses the host/DSO boundary as a FILE ARTIFACT both ++// sides load at boot: ++// TURBO_INNERQ_CALIBRATE=N + TURBO_INNERQ_OUT=path : gather K² over N blocks, ++// finalize, write `path`. (Cache discarded.) ++// TURBO_INNERQ_SCALE=path : load scale_inv at init, active from token 0. ++// Header-only (static-inline) so it adds no .cu to the build; the device state and ++// init/finalize live in set-rows.cu (the only TU needing device access). ++#pragma once ++ ++#include ++#include ++ ++#define INNERQ_MAX_CHANNELS 512 // max head_dim (Gemma-4 global = 512); Qwen = 128, Gemma SWA = 256 ++#define INNERQ_MAX_SLOTS 4 // distinct KV widths per model (Gemma-4 = 2: SWA+global) ++#define TURBO_INNERQ_MAGIC 0x31514954u // "TIQ1" little-endian (v1: single uniform head_dim) ++#define TURBO_INNERQ_MAGIC2 0x32514954u // "TIQ2" little-endian (v2: per-width slots) ++ ++// File layout: u32 magic, i32 count, float[count] scale_inv (host-native LE). ++static inline bool turbo_innerq_dump_file(const char * path, const float * scale_inv, int n) { ++ if (!path || n <= 0 || n > INNERQ_MAX_CHANNELS) return false; ++ FILE * f = fopen(path, "wb"); ++ if (!f) return false; ++ const uint32_t magic = TURBO_INNERQ_MAGIC; ++ const int32_t cnt = n; ++ bool ok = fwrite(&magic, sizeof(magic), 1, f) == 1 ++ && fwrite(&cnt, sizeof(cnt), 1, f) == 1 ++ && fwrite(scale_inv, sizeof(float), (size_t)n, f) == (size_t)n; ++ fclose(f); ++ return ok; ++} ++ ++// Reads scale_inv into out[0..n). Returns the count read, or -1 on error/mismatch. ++static inline int turbo_innerq_load_file(const char * path, float * out, int n) { ++ if (!path || n <= 0 || n > INNERQ_MAX_CHANNELS) return -1; ++ FILE * f = fopen(path, "rb"); ++ if (!f) return -1; ++ uint32_t magic = 0; int32_t cnt = 0; ++ if (fread(&magic, sizeof(magic), 1, f) != 1 || magic != TURBO_INNERQ_MAGIC || ++ fread(&cnt, sizeof(cnt), 1, f) != 1 || cnt <= 0 || cnt > n) { fclose(f); return -1; } ++ const size_t got = fread(out, sizeof(float), (size_t)cnt, f); ++ fclose(f); ++ return got == (size_t)cnt ? (int)cnt : -1; ++} ++ ++// ---- v2: per-width slots (head_dim varies per layer, e.g. Gemma-4 256-SWA + 512-global) ---- ++// Layout: u32 magic(TIQ2), i32 nslots, then per slot { i32 n_embd_gqa, i32 head_dim, ++// float[head_dim] scale_inv }. n_embd_gqa is the set-rows row width (ne00) that selects the ++// slot at runtime; 0 = wildcard (matches any width — used by the v1→v2 promotion below). ++static inline bool turbo_innerq_dump_file_v2(const char * path, int nslots, ++ const int * nembd, const int * hd, const float scale_inv[][INNERQ_MAX_CHANNELS]) { ++ if (!path || nslots <= 0 || nslots > INNERQ_MAX_SLOTS) return false; ++ FILE * f = fopen(path, "wb"); ++ if (!f) return false; ++ const uint32_t magic = TURBO_INNERQ_MAGIC2; ++ const int32_t ns = nslots; ++ bool ok = fwrite(&magic, sizeof(magic), 1, f) == 1 ++ && fwrite(&ns, sizeof(ns), 1, f) == 1; ++ for (int s = 0; s < nslots && ok; s++) { ++ const int32_t ne = nembd[s], h = hd[s]; ++ ok = h > 0 && h <= INNERQ_MAX_CHANNELS ++ && fwrite(&ne, sizeof(ne), 1, f) == 1 ++ && fwrite(&h, sizeof(h), 1, f) == 1 ++ && fwrite(scale_inv[s], sizeof(float), (size_t)h, f) == (size_t)h; ++ } ++ fclose(f); ++ return ok; ++} ++ ++// Loads per-slot scale_inv. Accepts both v2 (multi-slot) and v1 (single uniform → 1 wildcard ++// slot, n_embd_gqa=0). Returns nslots read, or -1 on error. out[s][0..hd_out[s]). ++static inline int turbo_innerq_load_file_v2(const char * path, int * nembd_out, int * hd_out, ++ float out[][INNERQ_MAX_CHANNELS]) { ++ if (!path) return -1; ++ FILE * f = fopen(path, "rb"); ++ if (!f) return -1; ++ uint32_t magic = 0; ++ if (fread(&magic, sizeof(magic), 1, f) != 1) { fclose(f); return -1; } ++ if (magic == TURBO_INNERQ_MAGIC2) { ++ int32_t ns = 0; ++ if (fread(&ns, sizeof(ns), 1, f) != 1 || ns <= 0 || ns > INNERQ_MAX_SLOTS) { fclose(f); return -1; } ++ for (int s = 0; s < ns; s++) { ++ int32_t ne = 0, h = 0; ++ if (fread(&ne, sizeof(ne), 1, f) != 1 || fread(&h, sizeof(h), 1, f) != 1 || ++ h <= 0 || h > INNERQ_MAX_CHANNELS || ++ fread(out[s], sizeof(float), (size_t)h, f) != (size_t)h) { fclose(f); return -1; } ++ nembd_out[s] = ne; hd_out[s] = h; ++ } ++ fclose(f); ++ return ns; ++ } ++ if (magic == TURBO_INNERQ_MAGIC) { // v1 single uniform → one wildcard slot ++ int32_t cnt = 0; ++ if (fread(&cnt, sizeof(cnt), 1, f) != 1 || cnt <= 0 || cnt > INNERQ_MAX_CHANNELS || ++ fread(out[0], sizeof(float), (size_t)cnt, f) != (size_t)cnt) { fclose(f); return -1; } ++ nembd_out[0] = 0; hd_out[0] = cnt; ++ fclose(f); ++ return 1; ++ } ++ fclose(f); ++ return -1; ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-wht-core.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-wht-core.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-wht-core.cuh +@@ -0,0 +1,67 @@ ++// opencoti-hook: turboquant perf-lift (Level B) — see docs/features/poly_kv.md (F5 M6-S2 #391) ++// ++// Pure-CUDA core for GGML_OP_TURBO_WHT: sign tables + the cooperative 128-pt Walsh–Hadamard ++// group rotation. Deliberately depends on NOTHING but CUDA built-ins (no ggml / common.cuh), ++// so the standalone L1 unit (.opencoti/l1-turbo-wht-unit.cu) compiles the IDENTICAL device ++// math the shipped kernel (turbo-wht.cu) runs — the round-trip / Parseval proof exercises the ++// real recipe, not a paraphrase. ++// ++// TURBO_WHT_S1/S2 and 1/√128 are byte-identical to TURBO3_WHT_SIGNS1/2 in turbo-dequant.cuh ++// (the convention the turbo K/V cache was quantised with). VERIFIED equal at build authoring ++// by .opencoti/l1-turbo-wht-unit.cu's table-diff against turbo-dequant.cuh. ++#pragma once ++ ++#define TURBO_WHT_INV_SQRT128 0.08838834764831845f // 1/sqrt(128) ++ ++static __constant__ float TURBO_WHT_S1[128] = { ++ -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, ++ -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, ++ 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, ++ -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, ++ -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f ++}; ++ ++static __constant__ float TURBO_WHT_S2[128] = { ++ 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, ++ 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, ++ 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, ++ -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, ++ 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, ++ -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f ++}; ++ ++// Cooperative 128-pt WHT over shared buf[0..127]; thread t owns buf[t]; all 128 threads ++// participate (no divergent early-return before the syncs). Returns lane t's rotated value; ++// the caller writes it out. direction 0 = forward (InnerQ ×scale_inv BEFORE signs1→WHT→ ++// 1/√128·signs2); direction 1 = inverse (signs2→WHT→1/√128·signs1, InnerQ ×scale_inv AFTER). ++// `ch` = InnerQ channel index for this lane (channel within head_dim); scale_inv may be ++// nullptr (pure orthonormal WHT). Butterfly is the turbo-cpy.cu validated `(t & h)==0` form. ++static __device__ __forceinline__ float turbo_wht_group128( ++ float * buf, int t, int direction, const float * scale_inv, int ch) { ++ const float * s_first = (direction == 0) ? TURBO_WHT_S1 : TURBO_WHT_S2; ++ const float * s_second = (direction == 0) ? TURBO_WHT_S2 : TURBO_WHT_S1; ++ ++ __syncthreads(); ++ if (direction == 0 && scale_inv) buf[t] *= scale_inv[ch]; ++ __syncthreads(); ++ buf[t] *= s_first[t]; ++ ++ for (int h = 1; h < 128; h <<= 1) { ++ __syncthreads(); ++ if ((t & h) == 0) { ++ const float a = buf[t], b = buf[t + h]; ++ buf[t] = a + b; ++ buf[t + h] = a - b; ++ } ++ } ++ __syncthreads(); ++ ++ float val = buf[t] * TURBO_WHT_INV_SQRT128 * s_second[t]; ++ if (direction == 1 && scale_inv) val *= scale_inv[ch]; ++ return val; ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cu b/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cu +@@ -0,0 +1,73 @@ ++// opencoti-hook: turboquant perf-lift (Level B) — see docs/features/poly_kv.md (F5 M6-S2 #391) ++// ++// GGML_OP_TURBO_WHT CUDA forward. One threadblock (128 threads) per 128-element WHT group. ++// Rotates the small n_q-sized Q (direction 0) / FA-output (direction 1) tensors — NOT the KV ++// cache (that stays compressed and is read in-register by the fused FA-VEC kernel at L2). The ++// device math lives in turbo-wht-core.cuh, shared verbatim with the standalone L1 unit so the ++// round-trip/Parseval proof exercises the real recipe. ++// ++// InnerQ is B-adapted: scale_inv comes from the device symbol (turbo_innerq_active_device_scale, ++// keyed on ne[0]=head_dim) unless a non-NULL src[1] (fork-compat) overrides it. Default-off / ++// width-miss ⇒ nullptr ⇒ pure orthonormal WHT, exactly as the Level-A dequant lift behaves. ++ ++#include "common.cuh" ++#include "turbo-wht.cuh" ++#include "turbo-innerq-dev.cuh" ++ ++static __global__ void k_turbo_wht_f32( ++ const float * __restrict__ src, float * __restrict__ dst, int direction, ++ int64_t head_dim, int64_t groups_per_head, int64_t n_groups, ++ const float * __restrict__ scale_inv) { ++ const int64_t g = (int64_t) blockIdx.x; ++ if (g >= n_groups) return; ++ const int t = threadIdx.x; ++ ++ const int64_t head_idx = g / groups_per_head; ++ const int64_t grp_in_head = g % groups_per_head; ++ const int64_t base = head_idx * head_dim + grp_in_head * 128; ++ ++ __shared__ float buf[128]; ++ buf[t] = src[base + t]; ++ ++ // InnerQ channel = position within head_dim = grp_in_head*128 + t (== base % head_dim + t). ++ const int ch = (int) (grp_in_head * 128 + t); ++ const float r = turbo_wht_group128(buf, t, direction, scale_inv, ch); ++ dst[base + t] = r; ++} ++ ++void ggml_cuda_turbo_wht(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * src = dst->src[0]; ++ GGML_ASSERT(src->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); ++ GGML_ASSERT(ggml_is_contiguous(src) && ggml_is_contiguous(dst)); ++ ++ int direction = 0, group_size = 0; ++ memcpy(&direction, (const char *) dst->op_params + 0, sizeof(int)); ++ memcpy(&group_size, (const char *) dst->op_params + sizeof(int), sizeof(int)); ++ GGML_ASSERT(group_size == 128 && "CUDA turbo_wht supports group_size 128 only (the WHT group)"); ++ ++ const int64_t head_dim = src->ne[0]; ++ GGML_ASSERT(head_dim % 128 == 0); ++ const int64_t n_elem = ggml_nelements(src); ++ const int64_t n_heads = n_elem / head_dim; ++ const int64_t groups_per_head = head_dim / 128; ++ const int64_t n_groups = groups_per_head * n_heads; ++ GGML_ASSERT(n_groups > 0 && n_groups < UINT_MAX); ++ ++ // B-adapted InnerQ: explicit src[1] graph tensor (fork-compat) wins; else the device symbol. ++ const float * d_scale_inv = nullptr; ++ if (dst->src[1]) { ++ d_scale_inv = (const float *) dst->src[1]->data; ++ } else { ++ const float * p = nullptr; ++ int hd = 0; ++ if (turbo_innerq_active_device_scale(head_dim, &p, &hd) && hd == (int) head_dim) { ++ d_scale_inv = p; ++ } ++ } ++ ++ const float * src_d = (const float *) src->data; ++ float * dst_d = (float *) dst->data; ++ cudaStream_t stream = ctx.stream(); ++ k_turbo_wht_f32<<<(unsigned) n_groups, 128, 0, stream>>>( ++ src_d, dst_d, direction, head_dim, groups_per_head, n_groups, d_scale_inv); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cuh +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-wht.cuh +@@ -0,0 +1,27 @@ ++// opencoti-hook: turboquant perf-lift (Level B) — see docs/features/poly_kv.md (F5 M6-S2 #391) ++// ++// GGML_OP_TURBO_WHT: a Walsh–Hadamard rotation op over f32, for the FUSED turbo FA-VEC ++// path (Level B). It runs on the small n_q-sized tensors — never on the KV cache: ++// direction 0 (forward, Q pre-rotation): [×scale_inv] → signs1 → WHT → 1/√128·signs2 ++// direction 1 (inverse, FA-output un-rotation): signs2 → WHT → 1/√128·signs1 → [×scale_inv] ++// ++// The sign tables (TURBO3_WHT_SIGNS1/2) and 1/√128 normalisation are EXACTLY the ++// convention the K/V cache was quantised with (turbo-dequant.cuh / set-rows.cu), so the ++// Parseval identity holds: forward-WHT(Q·scale_inv) · stored-rotated-K = Q_true·K_true. ++// ++// InnerQ is B-adapted (vs the AtomicBot fork's graph-tensor src[1]): the launcher ++// resolves scale_inv from the device symbol (turbo_innerq_active_device_scale, keyed on ++// ne[0]=head_dim); a non-NULL src[1] (fork-compat) takes precedence. Group size is 128 ++// (the WHT group; Gemma head_dim 256/512 = 2/4 groups). Acceptance = logit-equivalence. ++#pragma once ++ ++#include "common.cuh" ++ ++// Cooperative in-shared WHT over one 128-element group. All 128 threads of the block ++// participate (thread t owns buf[t]); no divergent early-return before the syncs. `ch` is ++// the InnerQ channel index for this lane (= channel within head_dim); scale_inv may be ++// nullptr (⇒ no InnerQ, pure orthonormal WHT — used by the round-trip unit & InnerQ-off). ++// Defined in turbo-wht-core.cuh so the shipped kernel and the standalone unit share it. ++#include "turbo-wht-core.cuh" ++ ++void ggml_cuda_turbo_wht(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +diff --git a/llama.cpp/ggml/src/ggml-quants.h b/llama.cpp/ggml/src/ggml-quants.h +--- a/llama.cpp/ggml/src/ggml-quants.h ++++ b/llama.cpp/ggml/src/ggml-quants.h +@@ -107,6 +107,31 @@ GGML_API void iq2xs_free_impl(enum ggml_type type); + GGML_API void iq3xs_init_impl(int grid_size); + GGML_API void iq3xs_free_impl(int grid_size); + ++// opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2) ++// TurboQuant KV cache compression (arXiv 2504.19874) ++GGML_API void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * GGML_RESTRICT y, int64_t k); ++GGML_API void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo3_0(const block_turbo3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo4_0(const block_turbo4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_turbo3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++GGML_API size_t quantize_turbo4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++GGML_API void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo2_0(const block_turbo2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_turbo2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++GGML_API void quantize_row_turbo8_0_ref(const float * GGML_RESTRICT x, block_turbo8_0 * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo8_0(const block_turbo8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_turbo8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++ ++// TQ3_1S: WHT-rotated 3-bit weight quantization (8-level Lloyd-Max) ++GGML_API void quantize_row_tq3_1s_ref(const float * GGML_RESTRICT x, block_tq3_1s * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_tq3_1s(const block_tq3_1s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_tq3_1s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++ ++// TQ4_1S: WHT-rotated 4-bit weight quantization (16-level Lloyd-Max) ++GGML_API void quantize_row_tq4_1s_ref(const float * GGML_RESTRICT x, block_tq4_1s * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_tq4_1s(const block_tq4_1s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_tq4_1s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++ + #ifdef __cplusplus + } + #endif +diff --git a/llama.cpp/ggml/src/ggml-turbo-quant.c b/llama.cpp/ggml/src/ggml-turbo-quant.c +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-turbo-quant.c +@@ -0,0 +1,1235 @@ ++/* ++ * TurboQuant: KV cache compression via PolarQuant + QJL ++ * Based on: arXiv 2504.19874 (ICLR 2026) ++ * ++ * Implements GGML_TYPE_TURBO2_0 (2-bit), GGML_TYPE_TURBO3_0 (3-bit) and ++ * GGML_TYPE_TURBO4_0 (4-bit) for use as --cache-type-k turboN in llama-server. ++ */ ++ ++#include "ggml-quants.h" ++#include "ggml-common.h" ++#include "ggml-impl.h" ++ ++#define _USE_MATH_DEFINES ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#ifndef M_PI ++#define M_PI 3.14159265358979323846 ++#endif ++ ++/* Global: WHT group size for CPU quantize path (set by CPU SET_ROWS handler) */ ++GGML_API int turbo3_cpu_wht_group_size = 0; ++ ++/* ---------- constants ---------- */ ++ ++#define TURBO_SEED_ROTATION 42 ++#define TURBO_SEED_QJL 1042 ++#define TURBO_D 128 /* rotation group size = head_dim (independent of block size) */ ++#define TURBO_QJL_CONST 1.2533141373155003f /* sqrt(pi/2) */ ++ ++/* Optimal centroids from paper (scaled by 1/sqrt(d)) */ ++/* 2-bit: {±0.453, ±1.51} / sqrt(d) */ ++static const float CENTROIDS_2BIT[4] = { -0.133462f, -0.039994f, 0.039994f, 0.133462f }; ++ ++/* 3-bit: Lloyd-Max for N(0, 1/128), pre-computed */ ++static const float CENTROIDS_3BIT[8] = { ++ -0.190685f, -0.117832f, -0.065717f, -0.021460f, ++ 0.021460f, 0.065717f, 0.117832f, 0.190685f ++}; ++ ++/* ---------- rotation matrix (lazy init) ---------- */ ++ ++static float turbo_rotation[TURBO_D * TURBO_D]; ++static float turbo_rotation_t[TURBO_D * TURBO_D]; /* transpose */ ++static int turbo_rotation_initialized = 0; ++ ++/* Simple LCG PRNG for deterministic rotation generation */ ++static uint64_t turbo_prng_state; ++ ++static void turbo_prng_seed(uint64_t seed) { ++ turbo_prng_state = seed; ++} ++ ++static double turbo_prng_normal(void) { ++ /* Box-Muller transform from uniform LCG */ ++ turbo_prng_state = turbo_prng_state * 6364136223846793005ULL + 1442695040888963407ULL; ++ double u1 = (double)(turbo_prng_state >> 11) / (double)(1ULL << 53); ++ if (u1 < 1e-15) u1 = 1e-15; ++ turbo_prng_state = turbo_prng_state * 6364136223846793005ULL + 1442695040888963407ULL; ++ double u2 = (double)(turbo_prng_state >> 11) / (double)(1ULL << 53); ++ return sqrt(-2.0 * log(u1)) * cos(2.0 * M_PI * u2); ++} ++ ++static void turbo_init_rotation(void) { ++ if (turbo_rotation_initialized) return; ++ ++ const int d = TURBO_D; ++ ++ /* Generate random Gaussian matrix */ ++ turbo_prng_seed(TURBO_SEED_ROTATION); ++ float G[TURBO_D * TURBO_D]; ++ for (int i = 0; i < d * d; i++) { ++ G[i] = (float)turbo_prng_normal(); ++ } ++ ++ /* QR decomposition via modified Gram-Schmidt */ ++ /* Q stored column-major in turbo_rotation */ ++ memcpy(turbo_rotation, G, d * d * sizeof(float)); ++ ++ for (int j = 0; j < d; j++) { ++ /* Normalize column j */ ++ float norm = 0.0f; ++ for (int i = 0; i < d; i++) { ++ norm += turbo_rotation[i * d + j] * turbo_rotation[i * d + j]; ++ } ++ norm = sqrtf(norm); ++ if (norm > 1e-10f) { ++ for (int i = 0; i < d; i++) { ++ turbo_rotation[i * d + j] /= norm; ++ } ++ } ++ ++ /* Orthogonalize remaining columns against j */ ++ for (int k = j + 1; k < d; k++) { ++ float dot = 0.0f; ++ for (int i = 0; i < d; i++) { ++ dot += turbo_rotation[i * d + j] * turbo_rotation[i * d + k]; ++ } ++ for (int i = 0; i < d; i++) { ++ turbo_rotation[i * d + k] -= dot * turbo_rotation[i * d + j]; ++ } ++ } ++ } ++ ++ /* Compute transpose */ ++ for (int i = 0; i < d; i++) { ++ for (int j = 0; j < d; j++) { ++ turbo_rotation_t[i * d + j] = turbo_rotation[j * d + i]; ++ } ++ } ++ ++ turbo_rotation_initialized = 1; ++} ++ ++/* ---------- QJL projection matrix (lazy init, seed-based) ---------- */ ++ ++static float turbo_qjl_matrix[TURBO_D * TURBO_D]; ++static float turbo_qjl_matrix_t[TURBO_D * TURBO_D]; ++static int turbo_qjl_initialized = 0; ++ ++static void turbo_init_qjl(void) { ++ if (turbo_qjl_initialized) return; ++ ++ const int d = TURBO_D; ++ turbo_prng_seed(TURBO_SEED_QJL); ++ ++ for (int i = 0; i < d * d; i++) { ++ turbo_qjl_matrix[i] = (float)turbo_prng_normal(); ++ } ++ ++ /* Transpose */ ++ for (int i = 0; i < d; i++) { ++ for (int j = 0; j < d; j++) { ++ turbo_qjl_matrix_t[i * d + j] = turbo_qjl_matrix[j * d + i]; ++ } ++ } ++ ++ turbo_qjl_initialized = 1; ++} ++ ++/* ---------- helper: matrix-vector multiply ---------- */ ++ ++static void matvec(const float * M, const float * x, float * y, int d) { ++ /* y = M @ x, M is row-major d×d */ ++ for (int i = 0; i < d; i++) { ++ float sum = 0.0f; ++ for (int j = 0; j < d; j++) { ++ sum += M[i * d + j] * x[j]; ++ } ++ y[i] = sum; ++ } ++} ++ ++/* ---------- nearest centroid ---------- */ ++ ++static int nearest_centroid_2bit(float val) { ++ /* Binary search on midpoints: {-0.133, -0.040, 0.040, 0.133} */ ++ if (val < -0.086728f) return 0; /* midpoint(-0.133, -0.040) */ ++ if (val < 0.000000f) return 1; /* midpoint(-0.040, 0.040) */ ++ if (val < 0.086728f) return 2; /* midpoint(0.040, 0.133) */ ++ return 3; ++} ++ ++static int nearest_centroid_3bit(float val) { ++ /* 8 centroids, find nearest via midpoints */ ++ if (val < -0.154259f) return 0; ++ if (val < -0.091775f) return 1; ++ if (val < -0.043589f) return 2; ++ if (val < 0.000000f) return 3; ++ if (val < 0.043589f) return 4; ++ if (val < 0.091775f) return 5; ++ if (val < 0.154259f) return 6; ++ return 7; ++} ++ ++static int nearest_centroid_4bit(float val) { ++ /* 16 centroids, optimal for N(0, 1/sqrt(128)), find nearest via midpoints */ ++ if (val < -0.145560f) return 0; ++ if (val < -0.103361f) return 1; ++ if (val < -0.079142f) return 2; ++ if (val < -0.060009f) return 3; ++ if (val < -0.043430f) return 4; ++ if (val < -0.028293f) return 5; ++ if (val < -0.013963f) return 6; ++ if (val < 0.000000f) return 7; ++ if (val < 0.013963f) return 8; ++ if (val < 0.028293f) return 9; ++ if (val < 0.043430f) return 10; ++ if (val < 0.060009f) return 11; ++ if (val < 0.079142f) return 12; ++ if (val < 0.103361f) return 13; ++ if (val < 0.145560f) return 14; ++ return 15; ++} ++ ++/* ---------- WHT sign arrays (must match CUDA/Metal, seed=42) ---------- */ ++ ++static const float turbo_cpu_s1[128] = { ++ -1,1,1,-1,-1,1,-1,1,-1,-1,1,1,1,1,1,1,1,-1,1,-1,1,-1,-1,1,1,1,-1,1,1,-1,-1,-1, ++ -1,1,1,-1,1,1,-1,1,-1,1,1,-1,-1,1,-1,1,1,1,1,-1,-1,-1,-1,-1,1,-1,1,1,1,1,-1,1, ++ -1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,1,-1,-1,1,1,1,-1,-1,1,1,-1,1,1,-1,1,-1, ++ -1,1,1,-1,1,-1,1,-1,1,1,1,1,-1,1,-1,1,1,-1,1,1,-1,-1,-1,-1,-1,1,1,-1,1,1,-1,1 ++}; ++ ++static const float turbo_cpu_s2[128] = { ++ 1,1,1,1,-1,1,1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,-1,-1,1,-1,1,-1,1,-1,-1,1,-1,1,1,1, ++ 1,1,-1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1,1,-1,1,-1,1,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1, ++ 1,-1,1,-1,-1,-1,-1,1,-1,1,-1,1,-1,-1,1,1,-1,1,-1,1,1,-1,1,-1,-1,-1,-1,1,-1,-1,1,-1, ++ 1,-1,1,1,1,-1,-1,1,-1,1,-1,1,1,-1,-1,1,-1,1,-1,1,1,-1,1,-1,1,-1,-1,-1,-1,-1,1,-1 ++}; ++ ++/* ---------- CPU forward WHT (in-place, group_size elements) ---------- */ ++ ++static void turbo_cpu_fwht(float * x, int group_size) { ++ const float * s1 = turbo_cpu_s1; ++ const float * s2 = turbo_cpu_s2; ++ const float inv_sqrt = (group_size == 128) ? 0.08838834764831845f : 0.125f; ++ ++ // signs1 ++ for (int i = 0; i < group_size; i++) x[i] *= s1[i]; ++ ++ // butterfly stages ++ for (int h = 1; h < group_size; h *= 2) { ++ for (int i = 0; i < group_size; i += h * 2) { ++ for (int j = i; j < i + h; j++) { ++ float a = x[j], b = x[j + h]; ++ x[j] = a + b; ++ x[j + h] = a - b; ++ } ++ } ++ } ++ ++ // normalize + signs2 ++ for (int i = 0; i < group_size; i++) x[i] *= inv_sqrt * s2[i]; ++} ++ ++/* ---------- CPU inverse WHT (in-place, group_size elements) ---------- ++ * ++ * Forward is y = D(s2) * N * H * D(s1) * x (N = 1/sqrt(group_size)) ++ * H is the unnormalized Hadamard butterfly with H*H = group_size * I, so ++ * (N*H) is self-inverse. s1 and s2 are ±1 diagonals, also self-inverse. ++ * The inverse therefore has the same structure with s1 and s2 swapped: ++ * x = D(s1) * N * H * D(s2) * y ++ */ ++GGML_API void turbo_cpu_fwht_inverse(float * x, int group_size) { ++ const float * s1 = turbo_cpu_s1; ++ const float * s2 = turbo_cpu_s2; ++ const float inv_sqrt = (group_size == 128) ? 0.08838834764831845f : 0.125f; ++ ++ // signs2 (undoes the s2 that was applied last in the forward pass) ++ for (int i = 0; i < group_size; i++) x[i] *= s2[i]; ++ ++ // butterfly stages (same as forward — self-inverse up to the inv_sqrt scaling below) ++ for (int h = 1; h < group_size; h *= 2) { ++ for (int i = 0; i < group_size; i += h * 2) { ++ for (int j = i; j < i + h; j++) { ++ float a = x[j], b = x[j + h]; ++ x[j] = a + b; ++ x[j + h] = a - b; ++ } ++ } ++ } ++ ++ // normalize + signs1 ++ for (int i = 0; i < group_size; i++) x[i] *= inv_sqrt * s1[i]; ++} ++ ++/* ---------- TURBO3_0: 3-bit PolarQuant with WHT rotation ---------- */ ++ ++void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TURBO3 == 0); ++ ++ // Read WHT group size from global (set by CPU SET_ROWS handler before each call). ++ // Fallback: 128 if row is 128-aligned, else 64. ++ extern int turbo3_cpu_wht_group_size; ++ int group_size = turbo3_cpu_wht_group_size; ++ if (group_size != 64 && group_size != 128) { ++ group_size = (k % 128 == 0) ? 128 : 64; ++ } ++ if (k % group_size != 0) group_size = (group_size == 128) ? 64 : 128; ++ assert(k % group_size == 0); ++ ++ const int n_groups = k / group_size; ++ const int blocks_per_group = group_size / QK_TURBO3; ++ ++ for (int g = 0; g < n_groups; g++) { ++ const float * grp_src = x + g * group_size; ++ block_turbo3_0 * grp_dst = y + g * blocks_per_group; ++ ++ // 1. L2 norm over the group ++ float norm_sq = 0.0f; ++ float buf[128]; // max group_size ++ for (int j = 0; j < group_size; j++) { ++ buf[j] = grp_src[j]; ++ norm_sq += buf[j] * buf[j]; ++ } ++ float grp_norm = sqrtf(norm_sq); ++ float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ // 2. Normalize ++ for (int j = 0; j < group_size; j++) buf[j] *= inv_norm; ++ ++ // 3. Forward WHT rotation ++ turbo_cpu_fwht(buf, group_size); ++ ++ // 4. Quantize + pack into sub-blocks ++ float recon_sq = 0.0f; ++ for (int b = 0; b < blocks_per_group; b++) { ++ block_turbo3_0 * blk = &grp_dst[b]; ++ const int off = b * QK_TURBO3; ++ ++ memset(blk->qs, 0, QK_TURBO3 / 4); ++ memset(blk->signs, 0, QK_TURBO3 / 8); ++ ++ for (int j = 0; j < QK_TURBO3; j++) { ++ int idx = nearest_centroid_3bit(buf[off + j]); ++ blk->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2); ++ if (idx & 0x4) { ++ blk->signs[j / 8] |= (1 << (j % 8)); ++ } ++ recon_sq += CENTROIDS_3BIT[idx] * CENTROIDS_3BIT[idx]; ++ } ++ } ++ ++ // 5. Corrected norm: grp_norm / recon_norm (matching CUDA kernel) ++ float recon_norm = sqrtf(recon_sq); ++ float corrected = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ for (int b = 0; b < blocks_per_group; b++) { ++ grp_dst[b].norm = GGML_FP32_TO_FP16(corrected); ++ } ++ } ++} ++ ++/* opencoti-hook: turboquant InnerQ (M6-S2) — CPU dequant per-channel un-equalize, ++ * PER-WIDTH (per-layer head_dim). Loads the TURBO_INNERQ_SCALE artifact once: v2 ++ * (magic "TIQ2" = 0x32514954) carries one slot per distinct KV width (n_embd_gqa → ++ * head_dim → scale_inv[head_dim]); v1 (magic "TIQ1" = 0x31514954) is a single uniform ++ * vector promoted to one wildcard slot (n_embd_gqa=0). Identity if unset/unreadable ⇒ ++ * default-off is byte-identical. The GPU set-rows forward equalizes K (path-shared); ++ * THIS un-equalize is the dequant-on-lift half of the bridge. ++ * ++ * FUSED-PATH INVARIANT (next milestone): this ×scale_inv un-equalize must stay isolated ++ * to dequantize_row_turbo3_0 (the lift's to_float). The fused attention path keeps K ++ * equalized and instead pre-scales Q by scale_inv (Q_pre·K_eq = Q·K), reading the SAME ++ * per-width table — so it must NOT route through this to_float (no double-correction). ++ * Do not move ×scale_inv into any code shared with the fused kernel. */ ++#define INNERQ_CPU_MAX 512 /* max head_dim (Gemma global = 512) */ ++#define INNERQ_CPU_MAX_SLOTS 4 /* distinct KV widths (Gemma-4 = 2: SWA+global) */ ++static int g_innerq_cpu_nslots = 0; ++static int g_innerq_cpu_nembd[INNERQ_CPU_MAX_SLOTS]; ++static int g_innerq_cpu_hd [INNERQ_CPU_MAX_SLOTS]; ++static float g_innerq_cpu_sinv [INNERQ_CPU_MAX_SLOTS][INNERQ_CPU_MAX]; ++static void turbo_innerq_cpu_load(void) { ++ static int loaded = 0; ++ if (loaded) return; ++ int ns = 0; ++ const char * path = getenv("TURBO_INNERQ_SCALE"); ++ if (path && path[0]) { ++ FILE * f = fopen(path, "rb"); ++ if (f) { ++ uint32_t magic = 0; ++ if (fread(&magic, sizeof(magic), 1, f) == 1) { ++ if (magic == 0x32514954u) { /* TIQ2: per-width slots */ ++ int32_t nslots = 0; ++ if (fread(&nslots, sizeof(nslots), 1, f) == 1 && ++ nslots > 0 && nslots <= INNERQ_CPU_MAX_SLOTS) { ++ int ok = 1; ++ for (int s = 0; s < nslots && ok; s++) { ++ int32_t ne = 0, hd = 0; ++ if (fread(&ne, sizeof(ne), 1, f) == 1 && fread(&hd, sizeof(hd), 1, f) == 1 && ++ hd > 0 && hd <= INNERQ_CPU_MAX && ++ fread(g_innerq_cpu_sinv[s], sizeof(float), (size_t)hd, f) == (size_t)hd) { ++ g_innerq_cpu_nembd[s] = ne; g_innerq_cpu_hd[s] = hd; ++ } else ok = 0; ++ } ++ if (ok) ns = nslots; ++ } ++ } else if (magic == 0x31514954u) { /* TIQ1: single uniform → wildcard slot */ ++ int32_t cnt = 0; ++ if (fread(&cnt, sizeof(cnt), 1, f) == 1 && cnt > 0 && cnt <= INNERQ_CPU_MAX && ++ fread(g_innerq_cpu_sinv[0], sizeof(float), (size_t)cnt, f) == (size_t)cnt) { ++ g_innerq_cpu_nembd[0] = 0; g_innerq_cpu_hd[0] = cnt; ns = 1; ++ } ++ } ++ } ++ fclose(f); ++ } ++ } ++ g_innerq_cpu_nslots = ns; /* 0 ⇒ identity (no InnerQ) */ ++ loaded = 1; /* publish only after the table is fully populated */ ++} ++/* Select the slot for a row of length k (= n_embd_gqa): exact width match, then a ++ * wildcard (n_embd_gqa==0) slot, else -1 (no InnerQ for this width). */ ++static int turbo_innerq_cpu_slot_for(int64_t k) { ++ for (int s = 0; s < g_innerq_cpu_nslots; s++) if (g_innerq_cpu_nembd[s] == (int)k) return s; ++ for (int s = 0; s < g_innerq_cpu_nslots; s++) if (g_innerq_cpu_nembd[s] == 0) return s; ++ return -1; ++} ++ ++void dequantize_row_turbo3_0(const block_turbo3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ // opencoti-hook: turboquant (M6-S2) — faithful CPU dequant (was a rotated-space ++ // stub that skipped the inverse WHT). Forward is: ++ // normalize → forward WHT (turbo_cpu_fwht) → quantize-to-centroid ++ // so the stored centroids are post-rotation, normalized values. The true-space ++ // inverse must therefore: reconstruct centroids → inverse WHT → denormalize (×norm). ++ // The previous `CENTROIDS_3BIT[idx]*norm` left the data WHT-rotated, which is only ++ // correct for a fused kernel that pre-rotates Q; for a plain dequant→f16→stock-FA ++ // path (and any offline round-trip) it is wrong. QK_TURBO3==128 is exactly one WHT ++ // group, so per-block inverse == per-group inverse. ++ assert(k % QK_TURBO3 == 0); ++ const int nb = k / QK_TURBO3; ++ // Per-width InnerQ: k == n_embd_gqa selects this layer's slot (head_dim + scale_inv). ++ // slot<0 ⇒ no InnerQ for this width (or default-off) ⇒ ×scale_inv is identity. ++ turbo_innerq_cpu_load(); ++ const int slot = turbo_innerq_cpu_slot_for(k); ++ const float * sinv = (slot >= 0) ? g_innerq_cpu_sinv[slot] : NULL; ++ const int hd = (slot >= 0) ? g_innerq_cpu_hd[slot] : 0; ++ for (int block = 0; block < nb; block++) { ++ float norm = GGML_FP16_TO_FP32(x[block].norm); ++ float buf[QK_TURBO3]; ++ for (int j = 0; j < QK_TURBO3; j++) { ++ uint8_t low2 = (x[block].qs[j/4] >> ((j%4)*2)) & 0x3; ++ uint8_t hi1 = (x[block].signs[j/8] >> (j%8)) & 0x1; ++ uint8_t idx = low2 | (hi1 << 2); ++ buf[j] = CENTROIDS_3BIT[idx]; ++ } ++ turbo_cpu_fwht_inverse(buf, QK_TURBO3); ++ for (int j = 0; j < QK_TURBO3; j++) { ++ // head_dim-aware channel: the dequant processes 128-blocks; folding the global ++ // element index by this layer's head_dim recovers the per-head channel (scale ++ // shared across heads). ×norm (block scale) then ×scale_inv[ch] (InnerQ) → true K. ++ const float si = (sinv && hd > 0) ? sinv[(int)(((int64_t)block * QK_TURBO3 + j) % hd)] : 1.0f; ++ y[block * QK_TURBO3 + j] = buf[j] * norm * si; ++ } ++ } ++} ++ ++size_t quantize_turbo3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO3 == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO3) * sizeof(block_turbo3_0); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo3_0_ref( ++ src + row * n_per_row, ++ (block_turbo3_0 *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++/* ---------- TURBO2_0: 2-bit PolarQuant (no QJL) ---------- */ ++ ++void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TURBO2 == 0); ++ ++ extern int turbo3_cpu_wht_group_size; ++ int group_size = turbo3_cpu_wht_group_size; ++ if (group_size != 64 && group_size != 128) { ++ group_size = (k % 128 == 0) ? 128 : 64; ++ } ++ if (k % group_size != 0) group_size = (group_size == 128) ? 64 : 128; ++ assert(k % group_size == 0); ++ ++ const int n_groups = k / group_size; ++ const int blocks_per_group = group_size / QK_TURBO2; ++ ++ for (int g = 0; g < n_groups; g++) { ++ const float * grp_src = x + g * group_size; ++ block_turbo2_0 * grp_dst = y + g * blocks_per_group; ++ ++ /* 1. L2 norm over the group */ ++ float norm_sq = 0.0f; ++ float buf[128]; ++ for (int j = 0; j < group_size; j++) { ++ buf[j] = grp_src[j]; ++ norm_sq += buf[j] * buf[j]; ++ } ++ float grp_norm = sqrtf(norm_sq); ++ float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ /* 2. Normalize */ ++ for (int j = 0; j < group_size; j++) buf[j] *= inv_norm; ++ ++ /* 3. Forward WHT rotation */ ++ turbo_cpu_fwht(buf, group_size); ++ ++ /* 4. Quantize + pack into sub-blocks */ ++ float recon_sq = 0.0f; ++ for (int b = 0; b < blocks_per_group; b++) { ++ block_turbo2_0 * blk = &grp_dst[b]; ++ const int off = b * QK_TURBO2; ++ ++ memset(blk->qs, 0, QK_TURBO2 / 4); ++ ++ for (int j = 0; j < QK_TURBO2; j++) { ++ int idx = nearest_centroid_2bit(buf[off + j]); ++ blk->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2); ++ recon_sq += CENTROIDS_2BIT[idx] * CENTROIDS_2BIT[idx]; ++ } ++ } ++ ++ /* 5. Corrected norm */ ++ float recon_norm = sqrtf(recon_sq); ++ float corrected = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ for (int b = 0; b < blocks_per_group; b++) { ++ grp_dst[b].norm = GGML_FP32_TO_FP16(corrected); ++ } ++ } ++} ++ ++void dequantize_row_turbo2_0(const block_turbo2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ // opencoti-hook: turboquant (M6-S2) — faithful 2-bit CPU dequant for the dequant-on-lift ++ // path (was a rotated-domain stub: CENTROIDS_2BIT[idx]*norm). 2-bit centroid → inverse WHT → ++ // ×norm → ×scale_inv(InnerQ), mirroring turbo3. QK_TURBO2==128 == one WHT group ⇒ per-block ++ // inverse == per-group. No QJL signs at 2-bit. Per-width InnerQ via k==n_embd_gqa slot. ++ assert(k % QK_TURBO2 == 0); ++ const int nb = k / QK_TURBO2; ++ turbo_innerq_cpu_load(); ++ const int slot = turbo_innerq_cpu_slot_for(k); ++ const float * sinv = (slot >= 0) ? g_innerq_cpu_sinv[slot] : NULL; ++ const int hd = (slot >= 0) ? g_innerq_cpu_hd[slot] : 0; ++ for (int block = 0; block < nb; block++) { ++ float norm = GGML_FP16_TO_FP32(x[block].norm); ++ float buf[QK_TURBO2]; ++ for (int j = 0; j < QK_TURBO2; j++) { ++ uint8_t idx = (x[block].qs[j/4] >> ((j%4)*2)) & 0x3; ++ buf[j] = CENTROIDS_2BIT[idx]; ++ } ++ turbo_cpu_fwht_inverse(buf, QK_TURBO2); ++ for (int j = 0; j < QK_TURBO2; j++) { ++ const float si = (sinv && hd > 0) ? sinv[(int)(((int64_t)block * QK_TURBO2 + j) % hd)] : 1.0f; ++ y[block * QK_TURBO2 + j] = buf[j] * norm * si; ++ } ++ } ++} ++ ++size_t quantize_turbo2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO2 == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO2) * sizeof(block_turbo2_0); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo2_0_ref( ++ src + row * n_per_row, ++ (block_turbo2_0 *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++/* ---------- TURBO8_0: 8-bit uniform int8 (WHT-rotated, near-lossless rung) ---------- */ ++/* opencoti-hook: turboquant tier family (M6-S2). Same normalize→forward-WHT→corrected-norm ++ * pipeline as turbo2/3/4, but quantizes each post-WHT value with a uniform signed int8 ++ * (q = clamp(round(x*TURBO8_SCALE), -127, 127); centroid = q/TURBO8_SCALE). 1 byte/value, no ++ * codebook, no QJL. block == group == 128 ⇒ per-block inverse WHT == per-group. */ ++ ++void quantize_row_turbo8_0_ref(const float * GGML_RESTRICT x, block_turbo8_0 * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TURBO8 == 0); ++ ++ extern int turbo3_cpu_wht_group_size; ++ int group_size = turbo3_cpu_wht_group_size; ++ if (group_size != 64 && group_size != 128) { ++ group_size = (k % 128 == 0) ? 128 : 64; ++ } ++ if (k % group_size != 0) group_size = (group_size == 128) ? 64 : 128; ++ assert(k % group_size == 0); ++ ++ const int n_groups = k / group_size; ++ const int blocks_per_group = group_size / QK_TURBO8; ++ ++ for (int g = 0; g < n_groups; g++) { ++ const float * grp_src = x + g * group_size; ++ block_turbo8_0 * grp_dst = y + g * blocks_per_group; ++ ++ float norm_sq = 0.0f; ++ float buf[128]; ++ for (int j = 0; j < group_size; j++) { ++ buf[j] = grp_src[j]; ++ norm_sq += buf[j] * buf[j]; ++ } ++ float grp_norm = sqrtf(norm_sq); ++ float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f; ++ ++ for (int j = 0; j < group_size; j++) buf[j] *= inv_norm; ++ ++ turbo_cpu_fwht(buf, group_size); ++ ++ float recon_sq = 0.0f; ++ for (int b = 0; b < blocks_per_group; b++) { ++ block_turbo8_0 * blk = &grp_dst[b]; ++ const int off = b * QK_TURBO8; ++ for (int j = 0; j < QK_TURBO8; j++) { ++ int q = (int) lroundf(buf[off + j] * TURBO8_SCALE); ++ if (q > 127) q = 127; ++ if (q < -127) q = -127; ++ blk->qs[j] = (int8_t) q; ++ const float c = (float) q / TURBO8_SCALE; ++ recon_sq += c * c; ++ } ++ } ++ ++ float recon_norm = sqrtf(recon_sq); ++ float corrected = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm; ++ for (int b = 0; b < blocks_per_group; b++) { ++ grp_dst[b].norm = GGML_FP32_TO_FP16(corrected); ++ } ++ } ++} ++ ++void dequantize_row_turbo8_0(const block_turbo8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ // int8 centroid (q/TURBO8_SCALE) → inverse WHT → ×norm → ×scale_inv(InnerQ). Mirrors turbo3. ++ assert(k % QK_TURBO8 == 0); ++ const int nb = k / QK_TURBO8; ++ turbo_innerq_cpu_load(); ++ const int slot = turbo_innerq_cpu_slot_for(k); ++ const float * sinv = (slot >= 0) ? g_innerq_cpu_sinv[slot] : NULL; ++ const int hd = (slot >= 0) ? g_innerq_cpu_hd[slot] : 0; ++ for (int block = 0; block < nb; block++) { ++ float norm = GGML_FP16_TO_FP32(x[block].norm); ++ float buf[QK_TURBO8]; ++ for (int j = 0; j < QK_TURBO8; j++) buf[j] = (float) x[block].qs[j] / TURBO8_SCALE; ++ turbo_cpu_fwht_inverse(buf, QK_TURBO8); ++ for (int j = 0; j < QK_TURBO8; j++) { ++ const float si = (sinv && hd > 0) ? sinv[(int)(((int64_t)block * QK_TURBO8 + j) % hd)] : 1.0f; ++ y[block * QK_TURBO8 + j] = buf[j] * norm * si; ++ } ++ } ++} ++ ++size_t quantize_turbo8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO8 == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO8) * sizeof(block_turbo8_0); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo8_0_ref( ++ src + row * n_per_row, ++ (block_turbo8_0 *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++/* ---------- TURBO4_0: 3-bit PolarQuant + 1-bit QJL ---------- */ ++ ++void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k) { ++ turbo_init_rotation(); ++ turbo_init_qjl(); ++ ++ assert(k % QK_TURBO4 == 0); ++ const int nb = k / QK_TURBO4; ++ const int d = QK_TURBO4; ++ ++ for (int block = 0; block < nb; block++) { ++ const float * src = x + block * d; ++ ++ /* Step 1: Extract norm */ ++ float norm_sq = 0.0f; ++ for (int i = 0; i < d; i++) norm_sq += src[i] * src[i]; ++ float norm = sqrtf(norm_sq); ++ ++ /* Normalize */ ++ float normalized[TURBO_D]; ++ if (norm > 1e-10f) { ++ const float inv = 1.0f / norm; ++ for (int i = 0; i < d; i++) normalized[i] = src[i] * inv; ++ } else { ++ memset(normalized, 0, d * sizeof(float)); ++ } ++ ++ /* Step 2: Forward WHT rotation (matches CUDA set_rows) */ ++ float rotated[TURBO_D]; ++ memcpy(rotated, normalized, d * sizeof(float)); ++ turbo_cpu_fwht(rotated, d); ++ ++#if TURBO4_USE_4BIT ++ /* Step 3: 4-bit quantization (16 centroids) */ ++ static const float CENTROIDS_4BIT[16] = { ++ -0.173926f, -0.117195f, -0.089527f, -0.068756f, ++ -0.051262f, -0.035597f, -0.020989f, -0.006938f, ++ 0.006938f, 0.020989f, 0.035597f, 0.051262f, ++ 0.068756f, 0.089527f, 0.117195f, 0.173926f ++ }; ++ uint8_t indices[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ indices[i] = (uint8_t)nearest_centroid_4bit(rotated[i]); ++ } ++ ++ /* Norm correction */ ++ float recon_norm_sq = 0.0f; ++ for (int i = 0; i < d; i++) { ++ recon_norm_sq += CENTROIDS_4BIT[indices[i]] * CENTROIDS_4BIT[indices[i]]; ++ } ++ float recon_norm = sqrtf(recon_norm_sq); ++ float corrected_norm = (recon_norm > 1e-10f) ? norm / recon_norm : norm; ++ y[block].norm = GGML_FP32_TO_FP16(corrected_norm); ++#else ++ /* Step 3: 3-bit quantization (8 centroids) */ ++ uint8_t indices[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ indices[i] = (uint8_t)nearest_centroid_3bit(rotated[i]); ++ } ++ ++ /* Step 4: Residual */ ++ float reconstructed[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ reconstructed[i] = CENTROIDS_3BIT[indices[i]]; ++ } ++ float mse_recon[TURBO_D]; ++ matvec(turbo_rotation_t, reconstructed, mse_recon, d); ++ ++ float residual[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ residual[i] = normalized[i] - mse_recon[i]; ++ } ++ ++ /* Step 5: QJL */ ++ float projected[TURBO_D]; ++ matvec(turbo_qjl_matrix, residual, projected, d); ++#endif ++ ++ /* Pack */ ++#if !TURBO4_USE_4BIT ++ y[block].norm = GGML_FP32_TO_FP16(norm); ++#endif ++ ++#if TURBO4_USE_4BIT ++ /* 4-bit PolarQuant: nibble pack into qs[64] */ ++ memset(y[block].qs, 0, d / 2); ++ for (int i = 0; i < d; i++) { ++ y[block].qs[i / 2] |= (uint8_t)((indices[i] & 0xF) << ((i % 2) * 4)); ++ } ++ y[block].rnorm = GGML_FP32_TO_FP16(0.0f); ++#else ++ /* Legacy 3-bit + QJL: pack 3-bit indices + QJL signs */ ++ memset(y[block].qs, 0, d * 3 / 8); ++ for (int i = 0; i < d; i++) { ++ int bit_offset = i * 3; ++ int byte_idx = bit_offset / 8; ++ int bit_pos = bit_offset % 8; ++ uint16_t val = (uint16_t)(indices[i] & 0x7); ++ y[block].qs[byte_idx] |= (uint8_t)(val << bit_pos); ++ if (bit_pos > 5 && byte_idx + 1 < d * 3 / 8) { ++ y[block].qs[byte_idx + 1] |= (uint8_t)(val >> (8 - bit_pos)); ++ } ++ } ++ memset(y[block].signs, 0, d / 8); ++ for (int i = 0; i < d; i++) { ++ if (projected[i] >= 0.0f) { ++ y[block].signs[i / 8] |= (1 << (i % 8)); ++ } ++ } ++#endif ++ } ++} ++ ++void dequantize_row_turbo4_0(const block_turbo4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ turbo_init_rotation(); ++ ++ assert(k % QK_TURBO4 == 0); ++ const int nb = k / QK_TURBO4; ++ const int d = QK_TURBO4; ++ ++#if TURBO4_USE_4BIT ++ /* opencoti-hook: turboquant (M6-S2) — faithful 4-bit CPU dequant for the dequant-on-lift ++ * path (was a rotated-domain stub: CENTROIDS_4BIT[idx]*norm, valid only for a fused kernel ++ * that pre-rotates Q). 4-bit centroid → inverse WHT → ×norm → ×scale_inv(InnerQ), exactly ++ * mirroring dequantize_row_turbo3_0. QK_TURBO4==128 == one WHT group ⇒ per-block inverse == ++ * per-group inverse. The forward (GPU k_set_rows_turbo4) writes {corrected_norm, nibbles}; ++ * this reconstructs TRUE K. Per-width InnerQ: k==n_embd_gqa selects the layer's slot. */ ++ static const float CENTROIDS_4BIT[16] = { ++ -0.173926f, -0.117195f, -0.089527f, -0.068756f, ++ -0.051262f, -0.035597f, -0.020989f, -0.006938f, ++ 0.006938f, 0.020989f, 0.035597f, 0.051262f, ++ 0.068756f, 0.089527f, 0.117195f, 0.173926f ++ }; ++ turbo_innerq_cpu_load(); ++ const int slot = turbo_innerq_cpu_slot_for(k); ++ const float * sinv = (slot >= 0) ? g_innerq_cpu_sinv[slot] : NULL; ++ const int hd = (slot >= 0) ? g_innerq_cpu_hd[slot] : 0; ++ for (int block = 0; block < nb; block++) { ++ float norm = GGML_FP16_TO_FP32(x[block].norm); ++ float buf[QK_TURBO4]; ++ for (int j = 0; j < d; j++) { ++ uint8_t idx = (x[block].qs[j / 2] >> ((j % 2) * 4)) & 0xF; ++ buf[j] = CENTROIDS_4BIT[idx]; ++ } ++ turbo_cpu_fwht_inverse(buf, QK_TURBO4); ++ for (int j = 0; j < d; j++) { ++ const float si = (sinv && hd > 0) ? sinv[(int)(((int64_t)block * QK_TURBO4 + j) % hd)] : 1.0f; ++ y[block * d + j] = buf[j] * norm * si; ++ } ++ } ++#else ++ /* Legacy 3-bit + QJL dequant */ ++ turbo_init_qjl(); ++ for (int block = 0; block < nb; block++) { ++ float norm = GGML_FP16_TO_FP32(x[block].norm); ++ ++ uint8_t indices[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ int bit_offset = i * 3; ++ int byte_idx = bit_offset / 8; ++ int bit_pos = bit_offset % 8; ++ uint16_t raw = (uint16_t)x[block].qs[byte_idx]; ++ if (byte_idx + 1 < d * 3 / 8) { ++ raw |= (uint16_t)x[block].qs[byte_idx + 1] << 8; ++ } ++ indices[i] = (uint8_t)((raw >> bit_pos) & 0x7); ++ } ++ ++ float signs[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ signs[i] = (x[block].signs[i / 8] & (1 << (i % 8))) ? 1.0f : -1.0f; ++ } ++ ++ float rnorm = GGML_FP16_TO_FP32(x[block].rnorm); ++ const float qjl_scale = TURBO_QJL_CONST / (float)d * rnorm; ++ ++ float rotated_recon[TURBO_D]; ++ for (int i = 0; i < d; i++) { ++ rotated_recon[i] = CENTROIDS_3BIT[indices[i]]; ++ } ++ float mse_recon[TURBO_D]; ++ matvec(turbo_rotation_t, rotated_recon, mse_recon, d); ++ ++ float qjl_recon[TURBO_D]; ++ matvec(turbo_qjl_matrix_t, signs, qjl_recon, d); ++ for (int i = 0; i < d; i++) { ++ qjl_recon[i] *= qjl_scale; ++ } ++ ++ float * dst = y + block * d; ++ for (int i = 0; i < d; i++) { ++ dst[i] = (mse_recon[i] + qjl_recon[i]) * norm; ++ } ++ } ++#endif ++} ++ ++size_t quantize_turbo4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO4 == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO4) * sizeof(block_turbo4_0); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo4_0_ref( ++ src + row * n_per_row, ++ (block_turbo4_0 *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++/* ================================================================== */ ++/* TQ3_1S / TQ4_1S: WHT-rotated weight quantization */ ++/* ================================================================== */ ++ ++/* Lloyd-Max centroids for N(0,1) — shared with Metal shaders */ ++static const float TQ3_0_CENTROIDS[8] = { ++ -1.996684f, -1.291398f, -0.740341f, -0.247508f, ++ 0.230106f, 0.725222f, 1.277503f, 1.988943f ++}; ++ ++static const float TQ4_0_CENTROIDS[16] = { ++ -2.732590f, -2.069017f, -1.618046f, -1.256231f, ++ -0.942340f, -0.656759f, -0.388048f, -0.128395f, ++ 0.128395f, 0.388048f, 0.656759f, 0.942340f, ++ 1.256231f, 1.618046f, 2.069017f, 2.732590f, ++}; ++ ++/* WHT sign pattern (golden ratio hash, 32-element blocks) — shared by TQ3 and TQ4 */ ++static const float TQ3_0_SIGNS[32] = { ++ +1.0f, -1.0f, +1.0f, -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, ++ -1.0f, -1.0f, +1.0f, -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, ++ -1.0f, -1.0f, +1.0f, -1.0f, +1.0f, -1.0f, -1.0f, +1.0f, ++ -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, -1.0f, -1.0f, +1.0f, ++}; ++ ++#define TQ_BLOCK_SIZE 32 ++#define TQ_INV_SQRT32 0.17677669529663688f /* 1/sqrt(32) */ ++ ++/* Forward RHT: sign flips -> WHT butterfly -> normalize */ ++static void tq3_0_rht_forward(float * buf) { ++ for (int i = 0; i < TQ_BLOCK_SIZE; i++) buf[i] *= TQ3_0_SIGNS[i]; ++ for (int step = 1; step < TQ_BLOCK_SIZE; step <<= 1) { ++ for (int i = 0; i < TQ_BLOCK_SIZE; i += step << 1) { ++ for (int j = i; j < i + step; j++) { ++ float a = buf[j], b = buf[j + step]; ++ buf[j] = a + b; ++ buf[j + step] = a - b; ++ } ++ } ++ } ++ for (int i = 0; i < TQ_BLOCK_SIZE; i++) buf[i] *= TQ_INV_SQRT32; ++} ++ ++/* Inverse RHT: WHT butterfly -> normalize + unsign */ ++static void tq3_0_rht_inverse(float * buf) { ++ for (int step = 1; step < TQ_BLOCK_SIZE; step <<= 1) { ++ for (int i = 0; i < TQ_BLOCK_SIZE; i += step << 1) { ++ for (int j = i; j < i + step; j++) { ++ float a = buf[j], b = buf[j + step]; ++ buf[j] = a + b; ++ buf[j + step] = a - b; ++ } ++ } ++ } ++ for (int i = 0; i < TQ_BLOCK_SIZE; i++) buf[i] *= TQ_INV_SQRT32 * TQ3_0_SIGNS[i]; ++} ++ ++/* Nearest centroid for TQ3 (8 centroids) */ ++static int tq3_0_choose_index(float val) { ++ /* Binary search on midpoints of TQ3_0_CENTROIDS */ ++ if (val < -1.644041f) return 0; ++ if (val < -1.015870f) return 1; ++ if (val < -0.493925f) return 2; ++ if (val < -0.008701f) return 3; ++ if (val < 0.477664f) return 4; ++ if (val < 1.001363f) return 5; ++ if (val < 1.633223f) return 6; ++ return 7; ++} ++ ++/* Nearest centroid for TQ4 (16 centroids) */ ++static int tq4_0_choose_index(float val) { ++ /* Binary search on midpoints of TQ4_0_CENTROIDS */ ++ if (val < -2.400804f) return 0; ++ if (val < -1.843532f) return 1; ++ if (val < -1.437139f) return 2; ++ if (val < -1.099286f) return 3; ++ if (val < -0.799550f) return 4; ++ if (val < -0.522404f) return 5; ++ if (val < -0.258222f) return 6; ++ if (val < 0.000000f) return 7; ++ if (val < 0.258222f) return 8; ++ if (val < 0.522404f) return 9; ++ if (val < 0.799550f) return 10; ++ if (val < 1.099286f) return 11; ++ if (val < 1.437139f) return 12; ++ if (val < 1.843532f) return 13; ++ if (val < 2.400804f) return 14; ++ return 15; ++} ++ ++/* ---------- TQ3_1S quantization ---------- */ ++ ++void quantize_row_tq3_1s_ref(const float * GGML_RESTRICT x, block_tq3_1s * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TQ3_0 == 0); ++ const int nb = k / QK_TQ3_0; ++ ++ for (int block = 0; block < nb; block++) { ++ const float * src_blk = x + block * QK_TQ3_0; ++ block_tq3_1s * blk = &y[block]; ++ ++ /* 1. Forward RHT */ ++ float buf[TQ_BLOCK_SIZE]; ++ memcpy(buf, src_blk, TQ_BLOCK_SIZE * sizeof(float)); ++ tq3_0_rht_forward(buf); ++ ++ /* 2. Split into two halves, compute RMS per half */ ++ float rms0 = 0.0f, rms1 = 0.0f; ++ for (int j = 0; j < 16; j++) rms0 += buf[j] * buf[j]; ++ for (int j = 16; j < 32; j++) rms1 += buf[j] * buf[j]; ++ rms0 = sqrtf(rms0 / 16.0f); ++ rms1 = sqrtf(rms1 / 16.0f); ++ ++ /* 3. Scale search (9 points) */ ++ static const float scales[] = { 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, 1.1f, 1.2f, 1.35f, 1.5f }; ++ float best_d0 = rms0, best_d1 = rms1; ++ float best_err = 1e30f; ++ ++ for (int si = 0; si < 9; si++) { ++ float d0 = rms0 * scales[si]; ++ float d1 = rms1 * scales[si]; ++ float inv0 = (d0 > 1e-10f) ? 1.0f / d0 : 0.0f; ++ float inv1 = (d1 > 1e-10f) ? 1.0f / d1 : 0.0f; ++ ++ float err = 0.0f; ++ for (int j = 0; j < 16; j++) { ++ int idx = tq3_0_choose_index(buf[j] * inv0); ++ float diff = buf[j] - TQ3_0_CENTROIDS[idx] * d0; ++ err += diff * diff; ++ } ++ for (int j = 16; j < 32; j++) { ++ int idx = tq3_0_choose_index(buf[j] * inv1); ++ float diff = buf[j] - TQ3_0_CENTROIDS[idx] * d1; ++ err += diff * diff; ++ } ++ if (err < best_err) { ++ best_err = err; ++ best_d0 = d0; ++ best_d1 = d1; ++ } ++ } ++ ++ /* 4. Iterative refinement (6 iterations) */ ++ for (int iter = 0; iter < 6; iter++) { ++ float inv0 = (best_d0 > 1e-10f) ? 1.0f / best_d0 : 0.0f; ++ float inv1 = (best_d1 > 1e-10f) ? 1.0f / best_d1 : 0.0f; ++ ++ float num0 = 0.0f, den0 = 0.0f; ++ float num1 = 0.0f, den1 = 0.0f; ++ for (int j = 0; j < 16; j++) { ++ int idx = tq3_0_choose_index(buf[j] * inv0); ++ float c = TQ3_0_CENTROIDS[idx]; ++ num0 += buf[j] * c; ++ den0 += c * c; ++ } ++ for (int j = 16; j < 32; j++) { ++ int idx = tq3_0_choose_index(buf[j] * inv1); ++ float c = TQ3_0_CENTROIDS[idx]; ++ num1 += buf[j] * c; ++ den1 += c * c; ++ } ++ if (den0 > 1e-10f) best_d0 = num0 / den0; ++ if (den1 > 1e-10f) best_d1 = num1 / den1; ++ } ++ ++ /* 5. Final quantize + pack */ ++ float inv0 = (best_d0 > 1e-10f) ? 1.0f / best_d0 : 0.0f; ++ float inv1 = (best_d1 > 1e-10f) ? 1.0f / best_d1 : 0.0f; ++ ++ blk->d0 = GGML_FP32_TO_FP16(best_d0); ++ blk->d1 = GGML_FP32_TO_FP16(best_d1); ++ memset(blk->qs, 0, QK_TQ3_0 * 3 / 8); ++ ++ /* TQ3 packing: 4 groups of 8 indices packed into 3 bytes each */ ++ for (int g = 0; g < 4; g++) { ++ uint8_t indices[8]; ++ for (int i = 0; i < 8; i++) { ++ int j = g * 8 + i; ++ float inv = (j < 16) ? inv0 : inv1; ++ indices[i] = (uint8_t)tq3_0_choose_index(buf[j] * inv); ++ } ++ uint8_t * qp = blk->qs + g * 3; ++ qp[0] = (indices[0] & 7) | ((indices[1] & 7) << 3) | ((indices[2] & 3) << 6); ++ qp[1] = ((indices[2] >> 2) & 1) | ((indices[3] & 7) << 1) | ((indices[4] & 7) << 4) | ((indices[5] & 1) << 7); ++ qp[2] = ((indices[5] >> 1) & 3) | ((indices[6] & 7) << 2) | ((indices[7] & 7) << 5); ++ } ++ } ++} ++ ++void dequantize_row_tq3_1s(const block_tq3_1s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TQ3_0 == 0); ++ const int nb = k / QK_TQ3_0; ++ ++ for (int blk_i = 0; blk_i < nb; blk_i++) { ++ float d0 = GGML_FP16_TO_FP32(x[blk_i].d0); ++ float d1 = GGML_FP16_TO_FP32(x[blk_i].d1); ++ ++ /* Unpack 3-bit indices */ ++ float buf[32]; ++ for (int g = 0; g < 4; g++) { ++ const uint8_t * qp = x[blk_i].qs + g * 3; ++ uint8_t idx[8]; ++ idx[0] = qp[0] & 7; ++ idx[1] = (qp[0] >> 3) & 7; ++ idx[2] = ((qp[0] >> 6) | (qp[1] << 2)) & 7; ++ idx[3] = (qp[1] >> 1) & 7; ++ idx[4] = (qp[1] >> 4) & 7; ++ idx[5] = ((qp[1] >> 7) | (qp[2] << 1)) & 7; ++ idx[6] = (qp[2] >> 2) & 7; ++ idx[7] = (qp[2] >> 5) & 7; ++ ++ for (int i = 0; i < 8; i++) { ++ int j = g * 8 + i; ++ float d = (j < 16) ? d0 : d1; ++ buf[j] = TQ3_0_CENTROIDS[idx[i]] * d; ++ } ++ } ++ ++ /* Inverse RHT */ ++ tq3_0_rht_inverse(buf); ++ ++ memcpy(y + blk_i * QK_TQ3_0, buf, QK_TQ3_0 * sizeof(float)); ++ } ++} ++ ++size_t quantize_tq3_1s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TQ3_0 == 0); ++ ++ size_t row_size = (n_per_row / QK_TQ3_0) * sizeof(block_tq3_1s); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_tq3_1s_ref( ++ src + row * n_per_row, ++ (block_tq3_1s *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++/* ---------- TQ4_1S quantization ---------- */ ++ ++void quantize_row_tq4_1s_ref(const float * GGML_RESTRICT x, block_tq4_1s * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TQ4_1S == 0); ++ const int nb = k / QK_TQ4_1S; ++ ++ for (int block = 0; block < nb; block++) { ++ const float * src_blk = x + block * QK_TQ4_1S; ++ block_tq4_1s * blk = &y[block]; ++ ++ /* 1. Forward RHT */ ++ float buf[TQ_BLOCK_SIZE]; ++ memcpy(buf, src_blk, TQ_BLOCK_SIZE * sizeof(float)); ++ tq3_0_rht_forward(buf); ++ ++ /* 2. Split into two halves, compute RMS per half */ ++ float rms0 = 0.0f, rms1 = 0.0f; ++ for (int j = 0; j < 16; j++) rms0 += buf[j] * buf[j]; ++ for (int j = 16; j < 32; j++) rms1 += buf[j] * buf[j]; ++ rms0 = sqrtf(rms0 / 16.0f); ++ rms1 = sqrtf(rms1 / 16.0f); ++ ++ /* 3. Scale search (9 points) */ ++ static const float scales[] = { 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, 1.1f, 1.2f, 1.35f, 1.5f }; ++ float best_d0 = rms0, best_d1 = rms1; ++ float best_err = 1e30f; ++ ++ for (int si = 0; si < 9; si++) { ++ float d0 = rms0 * scales[si]; ++ float d1 = rms1 * scales[si]; ++ float inv0 = (d0 > 1e-10f) ? 1.0f / d0 : 0.0f; ++ float inv1 = (d1 > 1e-10f) ? 1.0f / d1 : 0.0f; ++ ++ float err = 0.0f; ++ for (int j = 0; j < 16; j++) { ++ int idx = tq4_0_choose_index(buf[j] * inv0); ++ float diff = buf[j] - TQ4_0_CENTROIDS[idx] * d0; ++ err += diff * diff; ++ } ++ for (int j = 16; j < 32; j++) { ++ int idx = tq4_0_choose_index(buf[j] * inv1); ++ float diff = buf[j] - TQ4_0_CENTROIDS[idx] * d1; ++ err += diff * diff; ++ } ++ if (err < best_err) { ++ best_err = err; ++ best_d0 = d0; ++ best_d1 = d1; ++ } ++ } ++ ++ /* 4. Iterative refinement (6 iterations) */ ++ for (int iter = 0; iter < 6; iter++) { ++ float inv0 = (best_d0 > 1e-10f) ? 1.0f / best_d0 : 0.0f; ++ float inv1 = (best_d1 > 1e-10f) ? 1.0f / best_d1 : 0.0f; ++ ++ float num0 = 0.0f, den0 = 0.0f; ++ float num1 = 0.0f, den1 = 0.0f; ++ for (int j = 0; j < 16; j++) { ++ int idx = tq4_0_choose_index(buf[j] * inv0); ++ float c = TQ4_0_CENTROIDS[idx]; ++ num0 += buf[j] * c; ++ den0 += c * c; ++ } ++ for (int j = 16; j < 32; j++) { ++ int idx = tq4_0_choose_index(buf[j] * inv1); ++ float c = TQ4_0_CENTROIDS[idx]; ++ num1 += buf[j] * c; ++ den1 += c * c; ++ } ++ if (den0 > 1e-10f) best_d0 = num0 / den0; ++ if (den1 > 1e-10f) best_d1 = num1 / den1; ++ } ++ ++ /* 5. Final quantize + pack (nibble packing) */ ++ float inv0 = (best_d0 > 1e-10f) ? 1.0f / best_d0 : 0.0f; ++ float inv1 = (best_d1 > 1e-10f) ? 1.0f / best_d1 : 0.0f; ++ ++ blk->d0 = GGML_FP32_TO_FP16(best_d0); ++ blk->d1 = GGML_FP32_TO_FP16(best_d1); ++ memset(blk->qs, 0, QK_TQ4_1S / 2); ++ ++ for (int j = 0; j < QK_TQ4_1S; j++) { ++ float inv = (j < 16) ? inv0 : inv1; ++ int idx = tq4_0_choose_index(buf[j] * inv); ++ blk->qs[j / 2] |= (uint8_t)((idx & 0xF) << ((j & 1) * 4)); ++ } ++ } ++} ++ ++void dequantize_row_tq4_1s(const block_tq4_1s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ assert(k % QK_TQ4_1S == 0); ++ const int nb = k / QK_TQ4_1S; ++ ++ for (int blk_i = 0; blk_i < nb; blk_i++) { ++ float d0 = GGML_FP16_TO_FP32(x[blk_i].d0); ++ float d1 = GGML_FP16_TO_FP32(x[blk_i].d1); ++ ++ float buf[32]; ++ for (int j = 0; j < 32; j++) { ++ uint8_t idx = (x[blk_i].qs[j / 2] >> ((j & 1) * 4)) & 0xF; ++ float d = (j < 16) ? d0 : d1; ++ buf[j] = TQ4_0_CENTROIDS[idx] * d; ++ } ++ ++ /* Inverse RHT */ ++ tq3_0_rht_inverse(buf); ++ ++ memcpy(y + blk_i * QK_TQ4_1S, buf, QK_TQ4_1S * sizeof(float)); ++ } ++} ++ ++size_t quantize_tq4_1s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TQ4_1S == 0); ++ ++ size_t row_size = (n_per_row / QK_TQ4_1S) * sizeof(block_tq4_1s); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_tq4_1s_ref( ++ src + row * n_per_row, ++ (block_tq4_1s *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -674,6 +674,55 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { + .to_float = (ggml_to_float_t) dequantize_row_q1_0, + .from_float_ref = (ggml_from_float_t) quantize_row_q1_0_ref, + }, ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2) ++ [GGML_TYPE_TURBO3_0] = { ++ .type_name = "turbo3", ++ .blck_size = QK_TURBO3, ++ .type_size = sizeof(block_turbo3_0), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo3_0, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo3_0_ref, ++ }, ++ [GGML_TYPE_TURBO4_0] = { ++ .type_name = "turbo4", ++ .blck_size = QK_TURBO4, ++ .type_size = sizeof(block_turbo4_0), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo4_0, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo4_0_ref, ++ }, ++ [GGML_TYPE_TURBO2_0] = { ++ .type_name = "turbo2", ++ .blck_size = QK_TURBO2, ++ .type_size = sizeof(block_turbo2_0), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo2_0, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo2_0_ref, ++ }, ++ [GGML_TYPE_TURBO8_0] = { ++ .type_name = "turbo8", ++ .blck_size = QK_TURBO8, ++ .type_size = sizeof(block_turbo8_0), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo8_0, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo8_0_ref, ++ }, ++ [GGML_TYPE_TQ3_1S] = { ++ .type_name = "tq3_1s", ++ .blck_size = QK_TQ3_0, ++ .type_size = sizeof(block_tq3_1s), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_tq3_1s, ++ .from_float_ref = (ggml_from_float_t) quantize_row_tq3_1s_ref, ++ }, ++ [GGML_TYPE_TQ4_1S] = { ++ .type_name = "tq4_1s", ++ .blck_size = QK_TQ4_1S, ++ .type_size = sizeof(block_tq4_1s), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_tq4_1s, ++ .from_float_ref = (ggml_from_float_t) quantize_row_tq4_1s_ref, ++ }, + [GGML_TYPE_Q4_0] = { + .type_name = "q4_0", + .blck_size = QK4_0, +@@ -1081,9 +1130,10 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { + "MOE_FUSED_UP_GATE", + "STREAMING_FLASH_ATTN", + "FLASH_ATTN_TAIL_PARTIAL", ++ "TURBO_WHT", + }; + +-static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); ++static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); + + static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "none", +@@ -1194,9 +1244,10 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "moe_up_gate(up,gate,x)", + "streaming_flash_attn(q,k,v)", + "flash_attn_tail_partial(q,k,v)", ++ "turbo_wht(x)", + }; + +-static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99"); ++static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); + + static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); + +@@ -5577,6 +5628,40 @@ struct ggml_tensor * ggml_flash_attn_ext_tail_partial( + return result; + } + ++// opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. Walsh–Hadamard ++// rotation op for the fused turbo FA-VEC path (Q forward-rotate / FA-output ++// inverse-rotate). B-adapted vs the AtomicBot fork: `scale` is normally NULL and ++// the CUDA kernel resolves InnerQ from the device symbol (set-rows.cu), so no ++// memory-context graph-tensor plumbing is needed; a non-NULL `scale` (fork-compat) ++// is honoured as src[1] when supplied. Output shares a's shape/type (f32). ++struct ggml_tensor * ggml_turbo_wht( ++ struct ggml_context * ctx, ++ struct ggml_tensor * a, ++ int direction, ++ int group_size, ++ struct ggml_tensor * scale) { ++ GGML_ASSERT(ggml_is_contiguous(a)); ++ GGML_ASSERT(a->type == GGML_TYPE_F32); ++ GGML_ASSERT(direction == 0 || direction == 1); ++ ++ if (group_size == 0) { ++ group_size = (a->ne[0] % 128 == 0) ? 128 : 64; ++ } ++ GGML_ASSERT(group_size == 32 || group_size == 64 || group_size == 128); ++ GGML_ASSERT(a->ne[0] % group_size == 0); ++ ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, a->ne); ++ ++ result->op = GGML_OP_TURBO_WHT; ++ result->src[0] = a; ++ result->src[1] = scale; // InnerQ scale_inv (NULL ⇒ CUDA resolves from device symbol) ++ ++ memcpy((char *) result->op_params + 0, &direction, sizeof(int)); ++ memcpy((char *) result->op_params + sizeof(int), &group_size, sizeof(int)); ++ ++ return result; ++} ++ + void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec) { +@@ -7923,6 +8008,13 @@ size_t ggml_quantize_chunk( + case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2) ++ case GGML_TYPE_TURBO3_0: result = quantize_turbo3_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TURBO4_0: result = quantize_turbo4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TURBO2_0: result = quantize_turbo2_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TURBO8_0: result = quantize_turbo8_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TQ3_1S: result = quantize_tq3_1s(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TQ4_1S: result = quantize_tq4_1s(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_F16: + { + size_t elemsize = sizeof(ggml_fp16_t); +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -1988,6 +1988,57 @@ ggml_tensor * llm_graph_context::build_attn_mha( + ggml_tensor * v_mla, + float kq_scale, + int il) const { ++ // opencoti-hook: turboquant perf-lift (M6-S2 Level B) — see docs/features/poly_kv.md + ++ // plan lexical-gliding-flurry. FUSED FA-VEC turbo path: for turbo2/3/4 with K==V at ++ // head_dim ∈ {128,256} (the vec-instanced widths) we read the COMPRESSED cache in-register ++ // inside flash-attn instead of materializing it to f16. The expensive Walsh–Hadamard rotation ++ // then runs only on the n_q-sized Q (forward) and the small FA OUTPUT (inverse) — never on the ++ // KV cache. InnerQ ×scale_inv is folded into BOTH ggml_turbo_wht kernels via the device-resident ++ // scale symbol keyed on ne[0]=head_dim (B-adapted; no graph-tensor scale src). Parseval identity: ++ // WHT(Q⊘s)·K_stored == Q_true·K_true and WHT⁻¹(Σp·V_stored)⊘s == Σp·V_true, so attention is exact ++ // in true space (logit-equiv gate, NOT byte-equality). ++ // ++ // FALLBACK = Level-A materialize (the else-branch below): turbo8 (no vec tier), head_dim 512 ++ // global layers (no D=512 turbo vec instance), asymmetric K≠V, or the non-flash-attn branch — ++ // those still cast turbo→f16 up front (ggml_cuda_cpy → ggml_cpy_turbo_f16_cuda, WHT-once + ++ // InnerQ, VRAM-resident). A single Gemma model thus runs fused SWA layers + materialized global. ++ const auto is_turbo_vec = [](enum ggml_type t) { ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0; ++ }; ++ // opencoti-hook: turboquant perf-lift (M6-S2 Level B, L4.5 AB-prefill-hybrid). The fused FA-VEC ++ // turbo read wins at DECODE (in-register, bandwidth-bound: turbo2/3/4 = 56/51/48 tps vs 43 ++ // materialize) but LOSES at PREFILL (large n_q rides the vec kernel, not MMA: ~817-1537 vs ~2500 ++ // tps). So fuse only for decode-like ubatches (small per-stream query count); large prefill ++ // ubatches fall through to the Level-A materialize→f16→MMA path. Threshold is the per-stream ++ // query-token count = q->ne[2] / n_stream (decode=1, prefill=chunk). Env-tunable for the A/B: ++ // TBV_FUSE_MAX_NQ = max per-stream n_q that still fuses (default 8). Set huge → always-fuse ++ // (= L4 baseline); set 0 → always-materialize (= Level-A). No rebuild needed to sweep. ++ static const int tbv_fuse_max_nq = []() { ++ const char * e = getenv("TBV_FUSE_MAX_NQ"); ++ return e ? atoi(e) : 8; ++ }(); ++ const int64_t tbv_n_stream = k->ne[3] > 0 ? k->ne[3] : 1; ++ const int64_t tbv_nq_per_strm = q->ne[2] / tbv_n_stream; ++ const bool fused_turbo = ++ cparams.flash_attn && kq_b == nullptr && ++ is_turbo_vec(k->type) && k->type == v->type && ++ (k->ne[0] == 128 || k->ne[0] == 256) && ++ tbv_nq_per_strm <= tbv_fuse_max_nq; ++ ++ if (fused_turbo) { ++ // forward-rotate Q (×scale_inv → signs₁ → WHT → 1/√G·signs₂); group auto-picks 128 since ++ // head_dim%128==0 for both 128 and 256. Q reaches here as the contiguous projection reshape, ++ // but guard defensively (a rope'd view would trip the op's contiguity assert). ++ if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } ++ q = ggml_turbo_wht(ctx0, q, /*direction=*/0, /*group_size=*/0, /*scale=*/nullptr); ++ } else { ++ const auto is_turbo = [](enum ggml_type t) { ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_TURBO8_0; ++ }; ++ if (is_turbo(k->type)) { k = ggml_cast(ctx0, k, GGML_TYPE_F16); } ++ if (is_turbo(v->type)) { v = ggml_cast(ctx0, v, GGML_TYPE_F16); } ++ } ++ + const bool v_trans = v->nb[1] > v->nb[2]; + + // split the batch into streams if needed +@@ -2025,6 +2076,14 @@ ggml_tensor * llm_graph_context::build_attn_mha( + ggml_flash_attn_ext_add_sinks(cur, sinks); + ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32); + ++ if (fused_turbo) { ++ // opencoti-hook: turboquant perf-lift (M6-S2 Level B). Inverse-WHT + InnerQ un-equalize ++ // on the SMALL FA output (signs₂ → WHT → 1/√G·signs₁ → ×scale_inv), recovering true-space ++ // attention from the rotated/equalized cache. ne[0]=head_dim_v keys the InnerQ slot; for ++ // turbo K==V the per-width scale matches the one folded into the forward-Q rotation above. ++ cur = ggml_turbo_wht(ctx0, cur, /*direction=*/1, /*group_size=*/0, /*scale=*/nullptr); ++ } ++ + if (v_mla) { + #if 0 + // v_mla can be applied as a matrix-vector multiplication with broadcasting across dimension 3 == n_tokens. +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -143,6 +143,22 @@ + done + fi + ++ # 3b. opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392; see 0073-turboquant-kv ++ # in vendors/patches/llamafile/ + docs/protocols/UPSTREAM_SYNC.md. ++ # TurboQuant KV uses a FUSED in-register FA-VEC read (no dequant-on-lift), so the turbo ++ # K==V vec instances must compile in the DEFAULT build too — not just under FA_ALL_QUANTS. ++ # Omitting them leaves the DSO without ggml_cuda_flash_attn_ext_vec_case; ++ # dlopen then fails the instant turbo KV reaches flash-attention (bug: FA-VEC build-list, ++ # sibling of bug-339). Guarded so it is a no-op when FA_ALL_QUANTS=1 (the glob above ++ # already globbed these) and on a vanilla tree (the per-file [ -f ] test). ++ if [ "$fa_all_quants" != "1" ]; then ++ for f in "$ti_dir"/fattn-vec-instance-turbo2_0-turbo2_0.cu \ ++ "$ti_dir"/fattn-vec-instance-turbo3_0-turbo3_0.cu \ ++ "$ti_dir"/fattn-vec-instance-turbo4_0-turbo4_0.cu; do ++ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" ++ done ++ fi ++ + # 4. mmf instances (always included) + for f in "$ti_dir"/mmf-*.cu; do + [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" +@@ -228,6 +244,7 @@ + $llama_cpp_dir/ggml/src/ggml-neo-pipeline.cpp \ + $llama_cpp_dir/ggml/src/ggml-backend-meta.cpp \ + $llama_cpp_dir/ggml/src/ggml-quants.c \ ++ $llama_cpp_dir/ggml/src/ggml-turbo-quant.c \ + $llama_cpp_dir/ggml/src/ggml-threading.cpp" + + echo "Compiling core GGML sources..." diff --git a/patches/0075-d512-turbo-vec.patch b/patches/0075-d512-turbo-vec.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0ada55f31f8dcf2a43ca8d572d797c9128a9449 --- /dev/null +++ b/patches/0075-d512-turbo-vec.patch @@ -0,0 +1,121 @@ +From: opencoti +Subject: [PATCH 0075] F5 M6 PolyKV S2 — D=512 fused FA-VEC turbo2/turbo3 (Gemma-4 GLOBAL) + +#411. The Level-B fused FA-VEC turbo read (0073) covered head_dim {128,256} only; +the Gemma-4 A4B GLOBAL layers use head_dim 512 (key_length=512) and therefore fell +back to the Level-A turbo->f16 materialize — a decode tax that scales with the +global-cache n_kv. This patch adds the head_dim-512 fused FA-VEC instances for +turbo2/turbo3 so the global cache stays IN-REGISTER: + + - fattn-vec.cuh: extern DECL_FATTN_VEC_CASE(512, turbo2/3) (the kernel is already + D-generic: nthreads/nthreads_V derive from D arithmetically, KQ smem ~4-8KB, + VKQ ~16 regs at ncols=1 — well within budget; verified compile + run clean). + - fattn.cu: turbo kernel-selection relax (Q->ne[0]==512 routes to VEC) + the two + D=512 FATTN_VEC_CASE dispatch lines for turbo2/turbo3. + - template-instances/fattn-vec-instance-turbo{2,3}_0: emit the D=512 case; the + autogenerator (generate_cu_files.py) emits the same line so re-gen stays exact. + - llama-graph.cpp: build_attn_mha fuse gate extended (fused_turbo_512) so D=512 + turbo2/turbo3 decode ubatches fuse; turbo4/turbo8 @ 512 keep Level-A. + +Correctness is Parseval-EXACT: the WHT is block-diagonal over 128-blocks, so D=512 +decomposes into 4 independent Parseval-exact 128-blocks (WHT(Q/s).K_stored == +Q.K_true). Acceptance = LOGIT-EQUIVALENCE fused-vs-materialize (#355 harness), +real_frac=0.0 for BOTH turbo3 and turbo2 (bit-equivalent, not merely close). +Perf (the point): turbo3 @ ctx 27.7k, fused 41.0 vs materialize 29.7 tok/s = +38%, +scaling with n_kv. Gate: .opencoti/p411-d512-gate.sh (+ p411-perf.sh). + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -618,3 +618,8 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_0) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_0) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO4_0) ++ ++// opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY — Gemma-4 GLOBAL ++// layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256. ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -337,6 +337,12 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0) + ++ // opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL ++ // layers). turbo4/turbo8 keep no D=512 instance (Level-A materialize). Matches the fattn.cu ++ // kernel-selection relax + the L3 graph gate (both turbo2/3 + D==512 only). ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) ++ + GGML_ABORT("fatal error"); + } + +@@ -474,7 +480,13 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when + // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift. + if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0) { +- return can_use_vector_kernel ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE; ++ // opencoti #411 (F5 M6-S2 Level-B): turbo2/3 also ship a D=512 fused VEC instance for Gemma-4 ++ // GLOBAL layers (key_length=512) so the 256k global cache reads in-register instead of the ++ // Level-A turbo→f16 materialize. turbo4/turbo8 have no D=512 instance → keep the ≤256 cap ++ // (Level-A fallback at 512). The L3 graph gate only emits turbo2/3 K/V to FA at 512 to match. ++ const bool turbo512_vec = Q->ne[0] == 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 && ++ (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0); ++ return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE; + } + + // If Turing tensor cores are available, use them: +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu +@@ -6,3 +6,4 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512 +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu +@@ -6,3 +6,4 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512 +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +@@ -79,6 +79,10 @@ for type_k in TYPES_KV: + for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]: + with open(f"fattn-vec-instance-{get_short_name(type_t)}-{get_short_name(type_t)}.cu", "w") as f: + f.write(SOURCE_FATTN_VEC.format(type_k=type_t, type_v=type_t)) ++ # opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL ++ # layers, key_length=512). turbo4 has no D=512 instance (Level-A materialize fallback). ++ if type_t in ("GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0"): ++ f.write(f"DECL_FATTN_VEC_CASE(512, {type_t}, {type_t}); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512\n") + + for ncols in [8, 16, 32, 64]: + for ncols2 in [1, 2, 4, 8, 16, 32]: +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2019,10 +2019,17 @@ ggml_tensor * llm_graph_context::build_attn_mha( + }(); + const int64_t tbv_n_stream = k->ne[3] > 0 ? k->ne[3] : 1; + const int64_t tbv_nq_per_strm = q->ne[2] / tbv_n_stream; ++ // opencoti F5 M6-S2 Level-B #411: head_dim 512 (Gemma-4 GLOBAL layers, key_length=512) fuses ++ // for turbo2/turbo3 ONLY — the D=512 fused FA-VEC turbo instances added in #411 keep the 256k ++ // global cache in-register instead of the Level-A turbo→f16 materialize (a ~0.55× decode tax that ++ // scales with n_kv). turbo4/turbo8 at 512 still take the materialize fallback (no D=512 instance). ++ // The WHT is block-diagonal over 128-blocks, so 512 = 4 Parseval-exact 128-blocks → exact. ++ const bool fused_turbo_512 = ++ k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0); + const bool fused_turbo = + cparams.flash_attn && kq_b == nullptr && + is_turbo_vec(k->type) && k->type == v->type && +- (k->ne[0] == 128 || k->ne[0] == 256) && ++ (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) && + tbv_nq_per_strm <= tbv_fuse_max_nq; + + if (fused_turbo) { diff --git a/patches/0078-dca.patch b/patches/0078-dca.patch new file mode 100644 index 0000000000000000000000000000000000000000..917c08cdb9dd97235f7fb601a54ce91d35abad97 --- /dev/null +++ b/patches/0078-dca.patch @@ -0,0 +1,3515 @@ +From: opencoti +Subject: [PATCH 0078] DCA — Dual Chunk Attention for Gemma 4 A4B + Qwen2/3/3.5 (#593) + +Dual Chunk Attention (DCA) for long-context prefill on a single 3090, enabling the +Gemma 4 26B-A4B (iSWA) 1M-context path and the qwen2 / qwen3 / qwen3moe / qwen35 / +qwen35moe families. Chunked attention with per-regime RoPE (INTRA same-chunk causal, +SUCC prior chunk, INTER older chunks) lets a 262144-trained model serve far beyond +its native ctx-cap (the server slot honors >n_ctx_train when --dca on). + +The fused multi-chunk kernel dispatches on head_dim (#623): the DCA switch selects +dispatch_gqa for DKQ in {128 (qwen3), 256 (qwen35moe), 512 (gemma4)} instead of +the old hardcoded <512,512>, and the dca_fused path forces nstages<=1 + Q_in_reg=false +at all three kernel sites (iter, process_tile, host) so the narrower-head Qwen configs +(nstages_target=2, Q_in_reg=true natively) don't skip the K-load or read unpopulated Q +registers -> NaN. These clamps are no-ops on Gemma's DKQ=512 (native nstages_target=1, +Q_in_reg=false): the compiled kernel for the 512 path is byte-identical pre/post-fix. + +Flags (common/arg.cpp): --dca on|off, --dca-chunk-size N (default 8192), +--dca-yarn-factor F. A new LSE-emitting flash-attn op (GGML_OP_FLASH_ATTN_EXT_LSE) +streams the three regimes and combines them via log-sum-exp; the multi-chunk path +fuses all three key-tile phases into ONE kernel pass (no host LSE combine, no +streaming-LSE recompute). Two host-side optimisations keep the dense-mask cost +independent of n_kv: O6 derives the SUCC/INTER band bounds analytically (killing 2 +of 3 dense masks), and O9 narrows the INTRA mask to the single kernel-read window +[cq*c, (cq+1)*c) under analytical_bands (chunk_size % n_ubatch == 0). + +MEASURED on RTX 5000-class (bf16 KV, v7-coder Q4_K_M, NIAH needle): DCA-on retrieves +the needle at 2/4/8/16/32 chunks (16k..256k) and, after O9, prefill throughput at +256k is at parity with native (~1.0x, down from the pre-opt 1.59x). Scales to +512k/768k/1M (DCA-only — past the native cap). INERT when --dca off (the default): +the build is byte-identical to stock on the non-DCA path. + +qwen35 MTP loader (#622): the qwen35 arch is a Qwen3-Next hybrid that may carry an MTP +(nextn) head as the last block (e.g. Qwen3.6-27B-Omnimerge-v4: block_count=65, +nextn_predict_layers=1, blk.64 = MTP). The loader now reads nextn_predict_layers, +excludes the MTP block from the recurrent/transformer loops and the n_layer switch +(llama-model.cpp), loads blk..nextn.* instead of ssm/attn for the last +nextn_predict_layers blocks, and skips the MTP block in llm_build_qwen35 forward +(qwen35.cpp). llama-arch.cpp re-classifies the six NEXTN_* tensors +LLM_TENSOR_LAYER_OUTPUT -> LLM_TENSOR_LAYER_REPEATING (matching upstream): they are +stored per-block, so the loader must not fault on the block index. This makes the +qwen35 MTP model loadable and, with DCA on, the fused multi-chunk path over its 16 +hd256 full-attn layers is coherence-gate verified. + +Context-shift fragmentation fallback (#617B): #617's analytical INDEX bands +(chunk = key_index / chunk_size) assume cache cell-index == absolute position. A classic +context shift (seq_rm + seq_add past n_ctx) relabels pos[] in place WITHOUT compaction, +so index != pos permanently and the index-based kernel visits the wrong physical cells +(model collapses to a token-loop). A sticky is_fragmented latch (llama-kv-cells.h, set on +pos_add / cell-rm, cleared only on clear()) gates a dense POSITION-based SUCC/INTER+INTRA +mask fallback (set_input_dca fill + two extra op mask srcs, fattn-mma full-range-when- +masked): nullptr on the contiguous fast path (zero extra VRAM/compute, byte-identical to +the non-fragmented binary), dense n_kv-wide masks only once fragmented. Reachable only on +full-attention DCA archs (qwen2/3/3.5) generating past n_ctx; iSWA archs (gemma4) disable +KV-shift entirely so never hit it. + +Decode chunk-boundary fix (#635): the INTER-regime query position is 2*c, NOT c. With +#617's K pre-roped once at j%c, an INTER-Q roped at the constant c gives relative distance +c-(j%c) in [1,c], which OVERLAPS the recent/SUCC window (SUCC true rel runs to 2c-1) — a +chunk-end far key then masquerades as the most-recent token and floods ~50% of attention +mass at the first cq>=2 decode query, collapsing greedy decode ~2x before the model's +intrinsic limit. INTER-Q = 2*c puts every far key at rel [c+1,2c], strictly beyond the +window edge c: retrievable but unable to out-compete genuine recent keys. One host line +(set_input_dca in llama-kv-cache.cpp). Verified: greedy count-up matches the DCA-off +intrinsic break at chunk 256/512/1024 in BOTH decode modes, and a NIAH needle planted in +a far INTER chunk is still retrieved. Decode keeps the upstream stream-k path (an earlier +single-block-per-tile decode workaround was built on a falsified stream-k-FTZ-combine +hypothesis and has been reverted). + +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -199,6 +199,7 @@ LLAMA_SRCS_CPP := \ + llama.cpp/src/models/talkie.cpp \ + llama.cpp/src/models/wavtokenizer-dec.cpp \ + llama.cpp/src/models/xverse.cpp \ ++ llama.cpp/src/dca.cpp \ + llama.cpp/src/llama-adapter.cpp \ + llama.cpp/src/llama-arch.cpp \ + llama.cpp/src/llama-batch.cpp \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1505,6 +1505,34 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + else throw std::invalid_argument("--fused-moe-up-gate must be on|off"); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_FUSED_MOE_UP_GATE")); ++ // opencoti F5 dca — Dual Chunk Attention training-free long-context — see docs/features/gemma4_dca.md ++ add_opt(common_arg( ++ {"--dca"}, "MODE", ++ "Dual Chunk Attention (training-free long-context extension): off (default), on. " ++ "Routes full-attention layers (Gemma-4 global / Qwen all) through 3-regime chunked-position " ++ "attention so no query-key distance exceeds the pretrain window. Compose with " ++ "--rope-scaling yarn + --override-kv .context_length for context beyond native.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.dca_enabled = false; ++ else if (value == "on" || value == "true" || value == "1") params.dca_enabled = true; ++ else throw std::invalid_argument("--dca must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DCA")); ++ add_opt(common_arg( ++ {"--dca-chunk-size"}, "N", ++ "DCA chunk length in tokens. 0 = auto (the model's original/pretrain context window). " ++ "Intra-chunk positions are native; inter-chunk distances are clamped within the window.", ++ [](common_params & params, const std::string & value) { ++ params.dca_chunk_size = (uint32_t) std::stoul(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DCA_CHUNK_SIZE")); ++ add_opt(common_arg( ++ {"--dca-yarn-factor"}, "F", ++ "DCA YaRN mscale stretch composed with chunked attention. 1.0 = none (default).", ++ [](common_params & params, const std::string & value) { ++ params.dca_yarn_factor = std::stof(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DCA_YARN_FACTOR")); + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--pcie-autodetect"}, "MODE", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1645,6 +1645,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.neo_pipeline_mode = params.neo_pipeline_mode; + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md + cparams.fused_moe_up_gate = params.fused_moe_up_gate; ++ // opencoti F5 dca — see docs/features/gemma4_dca.md ++ cparams.dca_enabled = params.dca_enabled; ++ cparams.dca_chunk_size = params.dca_chunk_size; ++ cparams.dca_yarn_factor = params.dca_yarn_factor; + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -459,6 +459,11 @@ struct common_params { + // false = off (default; unfused 3-node path). true = fuse decode MoE + // up+gate+GLU into one GGML_OP_MOE_FUSED_UP_GATE op for the CUDA backend. + bool fused_moe_up_gate = false; ++ // opencoti F5 dca — Dual Chunk Attention training-free long-context (docs/features/gemma4_dca.md) ++ // off by default; chunk 0 = auto (<- n_ctx_orig_yarn); yarn_factor 1.0 = no YaRN stretch. ++ bool dca_enabled = false; ++ uint32_t dca_chunk_size = 0; ++ float dca_yarn_factor = 1.0f; + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -599,6 +599,8 @@ extern "C" { + + GGML_OP_TURBO_WHT, // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391 + ++ GGML_OP_FLASH_ATTN_EXT_LSE, // opencoti F5 dca — #593 (O+lse packed) ++ + GGML_OP_COUNT, + }; + +@@ -2487,6 +2489,64 @@ extern "C" { + float max_bias, + float logit_softcap); + ++ // opencoti F5 dca (#593) — flash-attention emitting NORMALIZED O packed with per-row ++ // LSE for the DCA 3-regime graph-math combine. dst = [DV+1, n_head, n_q, n_batch]; row ++ // [0,DV) = O, row [DV] = lse (= m + logf(sum exp), softcap+sinks folded in). Same ++ // (q,k,v,mask,scale,max_bias,logit_softcap) contract as ggml_flash_attn_ext. CUDA-only. ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): pos_q/chunk_size/regime select the analytical ++ // band-bounds path for the SUCC/INTER regimes — pass mask=NULL + the device I32 query-position ++ // vector pos_q (len q->ne[1]) + the DCA chunk_size + regime (1=SUCC, 2=INTER); the CUDA op derives ++ // the per-tile KV band on-device (no dense [n_kv x n_tokens] mask, no host O(n_kv*n_tokens) fill + ++ // H2D). INTRA (regime 0) keeps its native-causal mask: mask=, pos_q=NULL, chunk_size=0, ++ // regime=0 — the legacy form (src[4]=NULL, op_params[3..4]=0, byte-identical to pre-O6). ++ GGML_API struct ggml_tensor * ggml_flash_attn_ext_lse( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap, ++ struct ggml_tensor * pos_q, ++ int chunk_size, ++ int regime); ++ ++ // opencoti F5 dca perf (T87.pD Branch B) — FUSED single-pass DCA flash-attention. Replaces the ++ // 3 per-regime ggml_flash_attn_ext_lse ops + the host LSE combine with ONE FA pass that walks the ++ // full causal range [0,i] in 3 contiguous key-phases (INTER→SUCC→INTRA), each using its OWN ++ // pre-roped Q variant, with a single continuous online-softmax. The 3 regime bands PARTITION the ++ // causal range so this is bit-faithful to the combine (no double counting). Encoded as a ++ // GGML_OP_FLASH_ATTN_EXT_LSE node with regime=3 (FUSED); the CUDA forward branches on it. ++ // q_intra/q_succ/q_inter : the 3 pre-roped+permuted Q variants (same shape). ++ // k, v : roped-once K, V (as for ggml_flash_attn_ext). ++ // mask_intra : the native-causal INTRA mask (its KV_min/KV_max already bound the ++ // INTRA phase to [cq*c, causal_limit] via the O4 band-trim). ++ // mask_succ/mask_inter : opencoti F5 dca (#617B) FRAGMENTED-fallback dense masks. NULL on the ++ // contiguous fast path (the always-case ⇒ kernel uses analytical index ++ // bands, byte-identical to #617). Non-NULL only when the KV cache is ++ // non-contiguous after a context-shift (cell index != absolute pos): ++ // the kernel then runs each phase over the FULL key range and selects ++ // keys via these position-based masks (mutually exclusive across the 3). ++ // pos_q_real : device I32 [n_q] REAL abs query positions; kernel derives cq=pos/chunk. ++ // Output dst = [DV+1, n_head, n_q, n_batch] (same as the LSE op; the [DV] lse row is UNUSED — the ++ // single fused pass needs no combine). max_bias must be 0 (no ALiBi on the analytical bands). ++ GGML_API struct ggml_tensor * ggml_flash_attn_ext_dca_fused( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q_intra, ++ struct ggml_tensor * q_succ, ++ struct ggml_tensor * q_inter, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask_intra, ++ struct ggml_tensor * mask_succ, ++ struct ggml_tensor * mask_inter, ++ struct ggml_tensor * pos_q_real, ++ float scale, ++ float max_bias, ++ float logit_softcap, ++ int chunk_size); ++ + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); +diff --git a/llama.cpp/ggml/src/ggml-backend.cpp b/llama.cpp/ggml/src/ggml-backend.cpp +--- a/llama.cpp/ggml/src/ggml-backend.cpp ++++ b/llama.cpp/ggml/src/ggml-backend.cpp +@@ -1273,7 +1273,8 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra + // CUDA forward self-selects the per-tile kernel + resident fallback, and the + // S3c copy-suppression (~:1285) already handles its pinned-host K/V srcs. + if (*cur_backend_id == -1 && sched->n_backends > 1 && +- (node->op == GGML_OP_STREAMING_FLASH_ATTN || node->op == GGML_OP_MOE_FUSED_UP_GATE)) { ++ (node->op == GGML_OP_STREAMING_FLASH_ATTN || node->op == GGML_OP_MOE_FUSED_UP_GATE || ++ node->op == GGML_OP_FLASH_ATTN_EXT_LSE /* opencoti F5 dca #593 */)) { + *cur_backend_id = 0; + SET_CAUSE(node, "4.cuda-only"); + } +@@ -1638,6 +1639,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s + // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used + ggml_tensor * node = split->graph.nodes[0]; + if (split->graph.n_nodes > 0 && ++ input->buffer != NULL && // opencoti F5 dca: graph inputs may lack a buffer here; guard the usage query + ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(input->buffer) && ( + (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID) +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -2110,6 +2110,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + // owns the FFN nodes, so the CPU scheduler never reaches here. + GGML_ABORT("MOE_FUSED_UP_GATE has no CPU implementation (CUDA-only op)"); + } break; ++ case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 (CUDA-only, shares abort) + case GGML_OP_STREAMING_FLASH_ATTN: + { + // opencoti-hook: f5-rolling-kv — W4 M7-D #304. CUDA-only op. The +@@ -2402,6 +2403,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + // scheduled on CPU. Single task keeps the planner well-formed. + n_tasks = 1; + } break; ++ case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 (CUDA-only) + case GGML_OP_STREAMING_FLASH_ATTN: + { + // opencoti-hook: f5-rolling-kv — W4 M7-D #304. CUDA-only; never +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +@@ -476,6 +476,7 @@ static bool GGML_CALL ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev + // whose K/V srcs are pinned CUDA_Host buffers → abort. (MOE_FUSED was saved + // only by graph-hook placement; STREAMING_FLASH_ATTN under M7 stream-by- + // default is not — both fixed here for robustness.) ++ case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 (CUDA-only) + case GGML_OP_STREAMING_FLASH_ATTN: + case GGML_OP_MOE_FUSED_UP_GATE: + return false; +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -25,6 +25,7 @@ typedef void (* fattn_kernel_t)( + const char * __restrict__ mask, + const char * __restrict__ sinks, + const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -47,6 +48,16 @@ typedef void (* fattn_kernel_t)( + // op sets this immediately before dispatching one FA; launch_fattn drains it (sets it back to + // nullptr) so it never leaks to an unrelated FA. Defined in fattn.cu. See UPSTREAM_SYNC.md. + extern thread_local float * opencoti_fattn_dst_lse; ++// opencoti F5 dca perf (T87.pD-opt O2): launch_fattn sets this true iff a finalize kernel ran (so ++// dst_lse was populated), false on the in-kernel single-block path. Lets the DCA _lse op fall back to ++// streaming_lse_kernel only when the free emit didn't happen. Defined in fattn.cu. ++extern thread_local bool opencoti_fattn_dst_lse_written; ++// opencoti F5 dca perf (T87.pD-opt O6 Tier B): analytical band-bounds side-channel for DCA SUCC/INTER. ++// When pos_q != nullptr, launch_fattn fills KV_min/KV_max from pos_q[0]/chunk for the regime (no dense ++// mask scan; mask passed as nullptr) and drains these back. Defined in fattn.cu. See UPSTREAM_SYNC.md. ++extern thread_local const int * opencoti_fattn_dca_pos_q; ++extern thread_local int opencoti_fattn_dca_chunk; ++extern thread_local int opencoti_fattn_dca_regime; + + typedef float (*vec_dot_KQ_t)( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds); +@@ -645,10 +656,17 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { + } + } + ++// opencoti F5 dca perf (T87.pD-opt O4): this kernel emits BOTH the high edge (KV_max, the suffix-trim ++// upstream uses to skip fully-masked HIGH key-bands) AND a low edge (KV_min) so the FA kernels can also ++// skip fully-masked LOW key-bands. For ordinary causal masks KV_min == 0 (no-op, byte-identical); for ++// Dual Chunk Attention INTRA/SUCC regimes (valid band [lo,hi) starting above 0) and for sliding-window ++// masks, KV_min lets each query-tile attend only its contiguous band. KV_min may be nullptr (callers ++// that don't want the low trim), in which case only KV_max is written. + template + __launch_bounds__(FATTN_KQ_STRIDE/2, 1) + static __global__ void flash_attn_mask_to_KV_max( +- const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) { ++ const half2 * __restrict__ mask, int * __restrict__ KV_max, int * __restrict__ KV_min, ++ const int ne30, const int s31, const int s33) { + const int ne31 = gridDim.x; + const int tid = threadIdx.x; + const int sequence = blockIdx.y; +@@ -692,11 +710,79 @@ static __global__ void flash_attn_mask_to_KV_max( + // In either case, walk back the decrementation by FATTN_KQ_STRIDE. + KV_max_sj += FATTN_KQ_STRIDE; + ++ // opencoti F5 dca perf (T87.pD-opt O4): symmetric LOW-edge scan. Walk UP from the bottom tile and ++ // stop at the first tile that is not entirely -inf; its lower edge is the lowest KV position any ++ // query in this tile attends to. All threads must participate (warp_reduce + buf_iw) so this runs ++ // before the thread-0-only return below. Clamp to KV_max so a fully-masked row yields min==max (an ++ // empty [min,max) band → the FA kernel does no work, matching the existing KV_max==0 behaviour). ++ int KV_min_sj = 0; ++ if (KV_min != nullptr) { ++ for (; KV_min_sj < ne30 * FATTN_KQ_STRIDE; KV_min_sj += FATTN_KQ_STRIDE) { ++ int all_inf = 1; ++ ++#pragma unroll ++ for (int j = 0; j < ncols1; ++j) { ++ const float2 tmp = __half22float2(mask[j*s31 + KV_min_sj/2 + tid]); ++ all_inf = all_inf && int(isinf(tmp.x)) && int(isinf(tmp.y)); ++ } ++ ++ all_inf = warp_reduce_all(all_inf); ++ if (tid % WARP_SIZE == 0) { ++ buf_iw[tid / WARP_SIZE] = all_inf; ++ } ++ __syncthreads(); ++ all_inf = buf_iw[tid % WARP_SIZE]; ++ __syncthreads(); ++ all_inf = warp_reduce_all(all_inf); ++ ++ if (!all_inf) { ++ break; ++ } ++ } ++ if (KV_min_sj > KV_max_sj) { ++ KV_min_sj = KV_max_sj; ++ } ++ } ++ + if (threadIdx.x != 0) { + return; + } + + KV_max[sequence*ne31 + jt] = KV_max_sj; ++ if (KV_min != nullptr) { ++ KV_min[sequence*ne31 + jt] = KV_min_sj; ++ } ++} ++ ++// opencoti F5 dca perf (T87.pD-opt O6 Tier B): ANALYTICAL per-tile KV band producer for the DCA ++// SUCC/INTER regimes. Replaces the dense-mask scan (flash_attn_mask_to_KV_max) — there is no mask. The ++// SUCC/INTER band for query position p1 is a CONTIGUOUS causal key range fixed by cq = p1/chunk, and ++// (guard: chunk % n_ubatch == 0) cq is uniform across the whole ubatch, so a single [lo,hi) covers every ++// query tile. Key index == position (single-stream contiguous prefill), and chunk is a multiple of ++// FATTN_KQ_STRIDE so the edges are tile-aligned (exact). One thread per query tile jt writes that band; ++// the FA kernels clamp their key-tile loop to [KV_min, KV_max) (the O4 kb0_min / kb0_stop machinery). ++// SUCC (regime 1): [ (cq-1)*chunk , cq*chunk ) INTER (regime 2): [ 0 , (cq-1)*chunk ) ++// An empty band (e.g. INTER at cq==1 → hi==0 → lo==hi) ⇒ the FA does zero key-tiles ⇒ LSE = -inf, which ++// the host 3-regime combine weights as 0 (identical to the all-(-inf)-mask-row behaviour). n_stream==1 ++// for DCA so the flat thread index is exactly the (sequence==0) jt the kernels read as KV_*[0*iter_j+jt]. ++static __global__ void flash_attn_dca_bounds_to_KV( ++ const int * __restrict__ pos_q, int * __restrict__ KV_max, int * __restrict__ KV_min, ++ const int n_tiles, const int chunk, const int regime, const int n_kv_keys) { ++ const int jt = blockIdx.x * blockDim.x + threadIdx.x; ++ if (jt >= n_tiles) { ++ return; ++ } ++ const int p1 = pos_q[0]; // cq uniform across the ubatch (guard: chunk % n_ubatch == 0) ++ const int cq = p1 / chunk; ++ int lo, hi; ++ if (regime == 1) { lo = (cq - 1) * chunk; hi = cq * chunk; } // SUCC ++ else { lo = 0; hi = (cq - 1) * chunk; } // INTER (regime == 2) ++ if (lo < 0) { lo = 0; } ++ if (hi < 0) { hi = 0; } ++ if (hi > n_kv_keys) { hi = n_kv_keys; } ++ if (lo > hi) { lo = hi; } ++ KV_max[jt] = hi; ++ KV_min[jt] = lo; + } + + template // D == head size +@@ -1006,6 +1092,7 @@ void launch_fattn( + ggml_cuda_pool_alloc K_f16(pool); + ggml_cuda_pool_alloc V_f16(pool); + ggml_cuda_pool_alloc KV_max(pool); ++ ggml_cuda_pool_alloc KV_min(pool); // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + +@@ -1100,8 +1187,24 @@ void launch_fattn( + const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; + + KV_max.alloc(ne_KV_max); ++ KV_min.alloc(ne_KV_max); // opencoti F5 dca perf (T87.pD-opt O4): low-edge band, same shape as KV_max + flash_attn_mask_to_KV_max<<>> +- ((const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); ++ ((const half2 *) mask->data, KV_max.ptr, KV_min.ptr, iter_k, s31, s33); ++ CUDA_CHECK(cudaGetLastError()); ++ } else if (opencoti_fattn_dca_pos_q != nullptr && K->ne[1] % FATTN_KQ_STRIDE == 0) { ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): analytical band for a DCA SUCC/INTER regime — there ++ // is no mask to scan (mask==nullptr), so the mask-scan branch above never fires for these. Fill ++ // KV_min/KV_max from the query positions for ANY n_q (incl. decode n_q==1), so the band is always ++ // enforced. n_stream==1 for DCA ⇒ ne_KV_max == ntiles_x and the flat jt index is correct. ++ const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1); ++ const int ne_KV_max = blocks_num_KV_max.x * blocks_num_KV_max.y; ++ KV_max.alloc(ne_KV_max); ++ KV_min.alloc(ne_KV_max); ++ const int threads = 64; ++ const int blocks = (ne_KV_max + threads - 1) / threads; ++ flash_attn_dca_bounds_to_KV<<>>( ++ opencoti_fattn_dca_pos_q, KV_max.ptr, KV_min.ptr, ++ ne_KV_max, opencoti_fattn_dca_chunk, opencoti_fattn_dca_regime, (int) K->ne[1]); + CUDA_CHECK(cudaGetLastError()); + } + +@@ -1212,6 +1315,7 @@ void launch_fattn( + mask ? ((const char *) mask->data) : nullptr, + sinks ? ((const char *) sinks->data) : nullptr, + KV_max.ptr, ++ KV_min.ptr, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band (nullptr when prepass didn't run) + !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, logit_softcap, + Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], +@@ -1228,10 +1332,14 @@ void launch_fattn( + // loudly instead of silently corrupting the windowed-FA online-softmax combine. The in-kernel LSE + // emit (typedef-threaded MMA epilogue) is a separate prefill-overflow stage; on the decode path + // (stream_k always engaged) a fixup always runs. See docs/protocols/UPSTREAM_SYNC.md. ++ // opencoti F5 dca perf (T87.pD-opt O2): report (don't assert) whether a finalize kernel will run and ++ // thus whether dst_lse was populated. Callers that have a fallback (the DCA _lse op -> streaming_lse_ ++ // kernel) read opencoti_fattn_dst_lse_written and recompute only when false. The rolling-kv decode ++ // path always has stream_k engaged (fixup_ran true), so its behaviour is unchanged. + if (dst_lse) { + const bool fixup_ran = stream_k && (((int)blocks_num.x % ntiles_dst == 0 && (int)blocks_num.x > ntiles_dst) || (ntiles_dst % (int)blocks_num.x != 0)); + const bool combine_ran = !stream_k && parallel_blocks > 1; +- GGML_ASSERT((fixup_ran || combine_ran) && "rolling-kv: dst_lse requested but FA resolved to the in-kernel write path (no finalize kernel); unsupported on this path"); ++ opencoti_fattn_dst_lse_written = (fixup_ran || combine_ran); + } + + if (stream_k) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -527,9 +527,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( + } + } + ++ + template ++ typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ, ++ bool dca_fused = false> + static __device__ __forceinline__ void flash_attn_ext_f16_iter( + const float2 * const __restrict__ Q_f2, + const half2 * const __restrict__ K_h2, +@@ -565,8 +567,20 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols); + constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols); + constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols); +- constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); +- constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); ++ // opencoti F5 dca (#623): MUST mirror flash_attn_ext_f16_process_tile's forced Q_in_reg=false on the ++ // fused path. The config returns true for DKQ 128/256; if iter trusted it, iter would read Q from the ++ // Q_B register array (the Q_in_reg branch) — but Q_B is only populated under `if constexpr (!dca_fused)`, ++ // so on the DCA path it is uninitialised garbage -> NaN KQ -> '!' loop. Forcing false makes iter load Q ++ // from tile_Q (filled by dca_load_tile_Q), matching process_tile's layout. DKQ=512 already returns false. ++ constexpr bool Q_in_reg = dca_fused ? false : ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); ++ // opencoti F5 dca (#623): the dca_fused 3-phase path calls iter with last_iter=false and NO caller-side ++ // cp.async prologue, so iter must be self-contained per kb0 (load its own tile_K/mask/V). That only holds ++ // at nstages<=1. For narrow head dims (DKQ 128/256) the Ampere config returns nstages_target=2 (double- ++ // buffered: the caller is expected to prime tile_K and iter SKIPS the K-load at the `nstages<=1` guard ++ // below) -> tile_K read uninitialised -> NaN KQ -> garbage ('!' loop). DKQ=512 already gets nstages=1 ++ // (why Gemma's fused path works). Clamp the fused path to single-stage so 128/256 mirror the validated 512. ++ constexpr int nstages_cfg = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); ++ constexpr int nstages = dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg; + + constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_K = nbatch_K2 + 4; +@@ -593,7 +607,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + (V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup); + } else { + constexpr bool use_cp_async = nstages == 1; +- if (ncols2 > 1 || mask_h) { ++ if (mask_h) { // opencoti O6-B: null mask_h ⇒ band-only (KV_min/KV_max); stock ncols2>1 always has a mask + flash_attn_ext_f16_load_mask + (mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + } +@@ -692,7 +706,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + float KQ_rowsum_add[cols_per_thread] = {0.0f}; + + if constexpr (cols_per_warp == 8) { +- if (ncols2 > 1 || mask_h) { ++ if (mask_h) { // opencoti O6-B: null mask_h ⇒ band-only (KV_min/KV_max); stock ncols2>1 always has a mask + #pragma unroll + for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::I) { + const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::I; +@@ -754,7 +768,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + } + } + } else { // not Turing mma or T_B_KQ::I > 8 +- if (ncols2 > 1 || mask_h) { ++ if (mask_h) { // opencoti O6-B: null mask_h ⇒ band-only (KV_min/KV_max); stock ncols2>1 always has a mask + #pragma unroll + for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::J) { + const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::J; +@@ -916,6 +930,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + } + } + #endif // defined(TURING_MMA_AVAILABLE) ++ // opencoti DCA #635 probe B REMOVED — see the note where the debug globals used to live: decode is ++ // stream-k single-phase-per-block, so this in-kernel rescale point is not the SUCC->INTRA seam. + } + + // Convert KQ C tiles into B tiles for VKQ calculation: +@@ -939,7 +955,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + cp_async_wait_all(); + __syncthreads(); + if (!last_iter) { +- if (ncols2 > 1 || mask_h) { ++ if (mask_h) { // opencoti O6-B: null mask_h ⇒ band-only (KV_min/KV_max); stock ncols2>1 always has a mask + flash_attn_ext_f16_load_mask + (mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + } +@@ -1113,7 +1129,54 @@ template struct mma_tile_sizes { + }; + #endif // defined(TURING_MMA_AVAILABLE) + +-template ++// opencoti F5 dca perf (T87.pD Branch B): fill tile_Q SRAM from a Q source (one regime's pre-roped Q). ++// Extracted verbatim from process_tile's Q-load (the !Q_in_reg SRAM fill). The fused kernel calls this ++// once per phase to swap the live Q variant (Q_in_reg==false for D=512 ⇒ Q lives in tile_Q, not regs). ++// Caller controls __syncthreads (before: prior phase's tile_Q reads done; after: fill visible). ++template ++static __device__ __forceinline__ void dca_load_tile_Q( ++ half2 * const __restrict__ tile_Q, const float2 * const __restrict__ Q_f2, const float scale, ++ const int jt, const int zt_gqa, const uint3 ne01, const int gqa_ratio, ++ const int stride_Q1, const int stride_Q2) { ++ constexpr int warp_size = ggml_cuda_get_physical_warp_size(); ++ constexpr int ncols = ncols1 * ncols2; ++ constexpr int stride_tile_Q = DKQ/2 + 4; ++ const half2 scale_h2 = make_half2(scale, scale); ++#pragma unroll ++ for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) { ++ const int k0_start = stride_k == warp_size ? 0 : DKQ/2 - (DKQ/2) % (2*stride_k); ++ const int k0_stop = DKQ/2 - (DKQ/2) % (1*stride_k); ++ const int stride_jc = warp_size / stride_k; ++ if (k0_start == k0_stop) { ++ continue; ++ } ++#pragma unroll ++ for (int jc0 = 0; jc0 < ncols; jc0 += nwarps*stride_jc) { ++ const int jc = jc0 + threadIdx.y*stride_jc + (stride_k == warp_size ? 0 : threadIdx.x / stride_k); ++ if (jc0 + nwarps*stride_jc > ncols && jc >= ncols) { ++ break; ++ } ++ const int j = jc / ncols2; ++ const int c = jc % ncols2; ++ if ((ncols1 == 1 || jt*ncols1 + j < int(ne01.z)) && (ncols2 == 1 || zt_gqa*ncols2 + c < gqa_ratio)) { ++#pragma unroll ++ for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { ++ const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); ++ const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k]; ++ tile_Q[jc*stride_tile_Q + k] = scale_h2 * make_half2(tmp.x, tmp.y); ++ } ++ } else { ++#pragma unroll ++ for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { ++ const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); ++ tile_Q[jc*stride_tile_Q + k] = make_half2(0.0f, 0.0f); ++ } ++ } ++ } ++ } ++} ++ ++template + static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + const float2 * const __restrict__ Q_f2, + const half2 * const __restrict__ K_h2, +@@ -1137,7 +1200,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + const int jt, + const int zt_gqa, + const int kb0_start, +- const int kb0_stop) { ++ const int kb0_stop, ++ const int kb0_min, // opencoti F5 dca perf (T87.pD-opt O4): lowest non-masked k-block; loop starts here ++ const float2 * const __restrict__ Q_succ_f2 = nullptr, // opencoti F5 dca perf (Branch B fused): SUCC-phase pre-roped Q ++ const float2 * const __restrict__ Q_inter_f2 = nullptr, // INTER-phase pre-roped Q ++ const int dca_cq = 0, // chunk index cq (uniform per launch under analytical bands) ++ const int dca_c_blk = 0, // chunk_size / nbatch_fa (chunk width in k-blocks) ++ // opencoti F5 dca (#617B): FRAGMENTED-fallback dense masks. NULL ⇒ contiguous fast path (the ++ // analytical b1/b2 band partition below). Non-NULL (post context-shift, index!=pos) ⇒ each of ++ // the 3 phases runs the FULL owned [kb0_start,kb0_stop) and selects keys via its position-based ++ // mask (mutually exclusive across the 3 ⇒ each causal key contributes in exactly one phase). ++ const half * const __restrict__ mask_succ_h = nullptr, ++ const half * const __restrict__ mask_inter_h = nullptr) { + #if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + //In this kernel Q, K, V are matrices while i, j, k are matrix indices. + +@@ -1157,8 +1231,19 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2 (DKQ, DV, ncols); + constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols); + constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols); +- constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); +- constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); ++ // opencoti F5 dca (#623): the fused-DCA path swaps the 3 pre-roped Q variants by RELOADING tile_Q at ++ // each INTER/SUCC/INTRA phase boundary (see the dca_fused block below + dca_load_tile_Q), so the per- ++ // key iter MUST read Q from tile_Q — i.e. Q_in_reg==false. The config only returns false for DKQ=512 ++ // (Gemma 4); for DKQ=128/256 (qwen3 / qwen35moe) it returns true, which (a) routes the iter to the ++ // never-filled Q_B register array and (b) aliases tile_K onto tile_Q (line below), so the K-load ++ // clobbers the Q the phase loop just wrote — both corrupt the attention (observed: multi-chunk ++ // '!!!!'/garbage at hd128). Force false on the fused path at every head dim; no-op at DKQ=512. ++ constexpr bool Q_in_reg = dca_fused ? false : ggml_cuda_fattn_mma_get_Q_in_reg(DKQ, DV, ncols); ++ // opencoti F5 dca (#623): clamp the fused path to single-stage (see flash_attn_ext_f16_iter for the full ++ // rationale). nstages drives the tile_V/tile_mask shared-mem offsets just below (~:1126); the kernel, ++ // iter, and the host ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case smem sizing must all use this clamp. ++ constexpr int nstages_cfg = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); ++ constexpr int nstages = dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg; + + if (cols_per_warp > ncols) { + NO_DEVICE_CODE; +@@ -1196,6 +1281,135 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + KQ_max[col] = -FLT_MAX/2.0f; + } + ++ // opencoti F5 dca perf (T87.pD Branch B): single fused pass over the partitioned causal range. ++ // Sorted-key-axis phases: INTER [0,b1) Q_inter no-mask | SUCC [b1,b2) Q_succ no-mask | INTRA ++ // [b2,kb0_stop) Q_intra(=Q_f2)+causal mask. One continuous online-softmax (VKQ_C/KQ_max/KQ_rowsum ++ // persist across phases) ⇒ bit-faithful to the 3-op + host LSE combine (the bands partition [0,i]). ++ // Q swapped by reloading tile_Q at each boundary (Q_in_reg==false ⇒ iter reads Q from tile_Q). ++ // last_iter is dead at nstages<=1 (used only under nstages>1) ⇒ pass false everywhere; per-block ++ // k_VKQ_sup: full nbatch_fa for the whole-block INTER/SUCC bands, real ne11-bound for INTRA's tail. ++ if constexpr (dca_fused) { ++ if (mask_inter_h == nullptr) { ++ // ===== CONTIGUOUS FAST PATH (cell index == absolute position) — UNCHANGED from #617/#618 ===== ++ // Stream-k: this CUDA block owns key-blocks [kb0_start, kb0_stop) of the output tile. The three DCA ++ // phases partition the causal range — INTER [0,b1) | SUCC [b1,b2) | INTRA [b2,...] — so clip each to ++ // the owned sub-range and run only the non-empty intersection. kb0_start/kb0_stop/dca_cq are ++ // block-uniform (one tile per block, KV_max per-tile) ⇒ every thread takes the same phase guards, ++ // keeping the __syncthreads balanced. Partial (O, KQ_max, KQ_rowsum) over any contiguous k-range ++ // combine via the shared stream-k fixup regardless of which Q-phase produced them — the per-key ++ // pre-roped Q is baked into each partial, so the fixup is phase-agnostic. ++ int b1 = (dca_cq - 1) * dca_c_blk; if (b1 < 0) { b1 = 0; } ++ int b2 = dca_cq * dca_c_blk; if (b2 < 0) { b2 = 0; } ++ const int inter_lo = kb0_start; const int inter_hi = min(b1, kb0_stop); ++ const int succ_lo = max(kb0_start, b1); const int succ_hi = min(b2, kb0_stop); ++ const int intra_lo = max(kb0_start, b2); const int intra_hi = kb0_stop; ++ // Phase order is INTRA -> SUCC -> INTER (near keys FIRST). Online-softmax is order-invariant in ++ // exact math, so within ONE block this ordering is purely cosmetic; running the highest-logit ++ // current-chunk band first just keeps the running KQ_max high so later bands rescale cleanly. (The ++ // earlier "INTRA-first avoids an in-iter FTZ flush" rationale was FALSIFIED in #635: the real ++ // SOFTMAX_FTZ wipeout was in the CROSS-BLOCK stream-k fixup, not flash_attn_ext_f16_iter. The fix ++ // is to run DCA fused DECODE single-block-per-tile so there is no cross-block fixup at all — see the ++ // nblocks override in ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case. Each block still owns one ++ // contiguous [kb0_start,kb0_stop) band; the 3 sub-bands are just visited high-to-low.) ++ if (intra_hi > intra_lo) { // PHASE INTRA (same-chunk causal; mask supplies the within-last-block tail) ++ // Mirror the proven stock ncols2>1 path EXACTLY: oob_check=false, k_VKQ_sup=nbatch_fa. The ++ // INTRA mask (full n_kv width, -inf beyond the same-chunk causal diagonal) zeroes every key ++ // past the causal bound i — including the partial tail of the last block — just as it does ++ // for every non-DCA Gemma global layer. kb0_stop (from KV_max) already trims to the bound; ++ // the INTRA band starts at the exact block boundary b2 (chunk % nbatch_fa == 0). ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = intra_lo; kb0 < intra_hi; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ } ++ if (succ_hi > succ_lo) { // PHASE SUCC (immediately-prior chunk, Q_succ, band-only — no mask) ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_succ_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = succ_lo; kb0 < succ_hi; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_succ_f2, K_h2, V_h2, nullptr, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ } ++ if (inter_hi > inter_lo) { // PHASE INTER (older chunks, Q_inter, band-only — no mask) ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_inter_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = inter_lo; kb0 < inter_hi; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_inter_f2, K_h2, V_h2, nullptr, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ } ++ } else { ++ // ===== FRAGMENTED FALLBACK (opencoti F5 dca #617B) — cell index != absolute position ===== ++ // After a context-shift the cache is position-relabeled in place, so the analytical index bands ++ // above pick the WRONG physical cells. Instead run ALL THREE phases over the FULL owned range ++ // [kb0_start,kb0_stop); each phase applies its dense position-based mask (mask_inter/succ/intra) ++ // which the host filled mutually-exclusively (INTRA ck==cq, SUCC ck==cq-1, INTER ck SUCC -> INTER (near-keys-first) order as the contiguous fast path above. The dense ++ // masks are mutually exclusive so each causal key contributes in exactly one phase; within ONE block ++ // the online-softmax is order-invariant, so the ordering is cosmetic (see the falsified-FTZ note on ++ // the fast path). Decode also runs single-block-per-tile here (#635 nblocks override) so there is no ++ // cross-block fixup on the shifted-cache path either. ++ // PHASE INTRA (Q_intra=Q_f2, dense mask_intra=mask_h) ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ // PHASE SUCC (Q_succ, dense mask_succ) ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_succ_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_succ_f2, K_h2, V_h2, mask_succ_h, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ // PHASE INTER (Q_inter, dense mask_inter) ++ __syncthreads(); ++ dca_load_tile_Q(tile_Q, Q_inter_f2, scale, jt, zt_gqa, ne01, gqa_ratio, stride_Q1, stride_Q2); ++ __syncthreads(); ++ for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { ++ flash_attn_ext_f16_iter ++ ++ (Q_inter_f2, K_h2, V_h2, mask_inter_h, dstk, dstk_fixup, scale, slope, logit_softcap, ++ ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, ++ KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ } ++ } ++ } ++ ++ if constexpr (!dca_fused) { + // Load Q data into tile_Q, either temporarily or permanently. + // Q in registers is faster, but register pressure is the biggest bottleneck. + // The loading is done with decreasing granularity for D for better memory bandwidth. +@@ -1253,7 +1467,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + + __syncthreads(); + ++ // opencoti F5 dca perf (T87.pD-opt O4): start the K/V loop at the lowest non-masked k-block so ++ // fully-masked LOW key-bands (DCA INTRA/SUCC regimes, SWA windows) are skipped entirely. Clamp to ++ // kb0_stop-1 so the unconditional last iteration below stays in range; processing one extra masked ++ // block is the online-softmax identity (contributes nothing), and the cp_async preload below keys ++ // off this same kb0 so the multi-stage pipeline stays consistent. kb0_min==0 for causal masks (no-op). + int kb0 = kb0_start; ++ if (kb0_min > kb0) { ++ kb0 = kb0_min; ++ } ++ if (kb0_stop > 0 && kb0 > kb0_stop - 1) { ++ kb0 = kb0_stop - 1; ++ } + + // Preload mask and K data for first iteration when using cp_async with multiple stages: + if constexpr (nstages > 1) { +@@ -1261,7 +1486,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + constexpr bool use_cp_async = true; + constexpr bool oob_check = false; + constexpr int k_VKQ_sup = nbatch_fa; +- if (ncols2 > 1 || mask_h) { ++ if (mask_h) { // opencoti O6-B: null mask_h ⇒ band-only (KV_min/KV_max); stock ncols2>1 always has a mask + flash_attn_ext_f16_load_mask + (mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + } +@@ -1311,6 +1536,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, + KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); + } ++ } // end if constexpr (!dca_fused) + + // With multi-stage loading there is no __syncthreads at the end of the iter, + // there can be a race condition on shared memory access for combining/writing back results. +@@ -1695,7 +1921,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup, + scale, slope, logit_softcap, ne01, ne02, gqa_ratio, + stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, +- jt, kb0_start, kb0_stop); ++ jt, kb0_start, kb0_stop, kb0_min); + NO_DEVICE_CODE; + #endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + } +@@ -1709,6 +1935,7 @@ static __global__ void flash_attn_ext_f16( + const char * __restrict__ mask, + const char * __restrict__ sinks, + const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -1806,7 +2033,7 @@ static __global__ void flash_attn_ext_f16( + + const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q); + const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV); +- const half * mask_h = ncols2 == 1 && !mask ? nullptr : ++ const half * mask_h = !mask ? nullptr : + (const half *) (mask + nb33*(sequence % ne33)); + float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2); + +@@ -1818,17 +2045,20 @@ static __global__ void flash_attn_ext_f16( + if (KV_max) { + kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); + } ++ // opencoti F5 dca perf (T87.pD-opt O4): lowest non-masked k-block for this query-tile (0 when no ++ // low trim, i.e. causal masks → unchanged). process_tile starts its K/V loop here. ++ const int kb0_min = KV_min ? (KV_min[sequence*iter_j + jt] / nbatch_fa) : 0; + constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. + if (kb0_start == 0) { + constexpr bool needs_fixup = false; // CUDA block is working on an entire tile. + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, +- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + } else { + constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile. + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, +- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + } + + kbc += iter_k; +@@ -1852,7 +2082,7 @@ static __global__ void flash_attn_ext_f16( + + const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q); + const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV); +- const half * mask_h = ncols2 == 1 && !mask ? nullptr : ++ const half * mask_h = !mask ? nullptr : + (const half *) (mask + nb33*(sequence % ne33)); + float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2); + +@@ -1864,14 +2094,16 @@ static __global__ void flash_attn_ext_f16( + if (KV_max) { + kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); + } ++ // opencoti F5 dca perf (T87.pD-opt O4): lowest non-masked k-block for this query-tile (see above). ++ const int kb0_min = KV_min ? (KV_min[sequence*iter_j + jt] / nbatch_fa) : 0; + + constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. + constexpr bool needs_fixup = false; + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, +- ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -1956,6 +2188,355 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + } + + ++// opencoti F5 dca perf (T87.pD Branch B + stream-k): FUSED single-pass DCA kernel. Uses the stock kbc ++// stream-k decomposition (load-balances the causal triangle ⇒ dst_meta partials + host fixup), walking ++// the partitioned causal range in INTER→SUCC→INTRA phases via process_tile<...,dca_fused=true> which ++// clips each phase to the block's owned [kb0_start,kb0_stop). Extra params vs the stock kernel: Q_succ/ ++// Q_inter (pre-roped Q for the older-chunk phases), pos_q (I32 real abs query pos), chunk (tokens). ++template ++__launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2)) ++static __global__ void flash_attn_ext_f16_dca_fused( ++ const char * __restrict__ Q, ++ const char * __restrict__ K, ++ const char * __restrict__ V, ++ const char * __restrict__ mask, ++ const char * __restrict__ mask_succ, // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ analytical fast path) ++ const char * __restrict__ mask_inter, // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ analytical fast path) ++ const char * __restrict__ sinks, // unused (DCA has no attention sinks) ++ const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // unused in fused (per-phase bounds derived analytically) ++ const char * __restrict__ Q_succ, ++ const char * __restrict__ Q_inter, ++ const int * __restrict__ pos_q, ++ const int chunk, ++ float * __restrict__ dst, ++ float2 * __restrict__ dst_meta, // unused (single block per tile) ++ const float scale, ++ const float max_bias, ++ const float m0, ++ const float m1, ++ const uint32_t n_head_log2, ++ const float logit_softcap, ++ const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03, ++ const int32_t nb01, const int32_t nb02, const int32_t nb03, ++ const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13, ++ const int32_t nb11, const int32_t nb12, const int64_t nb13, ++ const int32_t nb21, const int32_t nb22, const int64_t nb23, ++ const int32_t ne31, const int32_t ne32, const int32_t ne33, ++ const int32_t nb31, const int32_t nb32, const int64_t nb33) { ++#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)) ++ if (use_logit_softcap && !(DKQ == 128 || DKQ == 256 || DKQ == 512)) { NO_DEVICE_CODE; return; } ++ ++ constexpr int warp_size = ggml_cuda_get_physical_warp_size(); ++ constexpr int ncols = ncols1 * ncols2; ++ constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols); ++ constexpr int nthreads = ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols); ++ constexpr int nwarps = nthreads / warp_size; ++ ++ const int gqa_ratio = ne02 / ne12; ++ const int stride_Q1 = nb01 / sizeof(float2); ++ const int stride_Q2 = nb02 / sizeof(float2); ++ const int stride_K = nb11 / sizeof(half2); ++ const int stride_mask = nb31 / sizeof(half); ++ const int stride_V = V_is_K_view ? stride_K : nb21 / sizeof(half2); ++ ++ const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa; ++ const int iter_j = (int(ne01.z) + (ncols1 - 1)) / ncols1; ++ const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2; ++ ++ // Stream-k decomposition (verbatim from the stock flash_attn_ext_f16): the fixed grid of CUDA blocks ++ // is mapped onto (ntiles_dst × iter_k) work via the flat kbc index, so each block owns a contiguous ++ // [kb0_start, kb0_stop) slice of one or more output tiles. This load-balances the causal triangle ++ // (late-query tiles do ~iter_k k-iters, early ones ~1) that one-block-per-tile could not. The only ++ // DCA delta vs stock: per tile we also resolve Q_succ_f2/Q_inter_f2 + the chunk index dca_cq, and call ++ // process_tile<...,dca_fused=true> which clips its 3 phases to the owned k-range. dst_meta carries the ++ // partial (O,max,rowsum); the host launch runs the shared stream-k fixup to combine seams. ++ const int dca_c_blk = chunk / nbatch_fa; // chunk width in k-blocks (loop-invariant) ++ ++ int kbc = int64_t(blockIdx.x + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; ++ const int kbc_stop = int64_t(blockIdx.x + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; ++ ++ int kb0_start = kbc % iter_k; ++ int kb0_stop = min(iter_k, kb0_start + kbc_stop - kbc); ++ ++ while (kbc < kbc_stop && kb0_stop == iter_k) { ++ const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12); ++ const int z_KV = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa); ++ const int zt_gqa = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j); ++ const int jt = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k; ++ const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; ++ ++ const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q); ++ const float2 * Q_succ_f2 = (const float2 *) (Q_succ + nb03*sequence + nb02*zt_Q); ++ const float2 * Q_inter_f2 = (const float2 *) (Q_inter + nb03*sequence + nb02*zt_Q); ++ const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV); ++ const half * mask_h = !mask ? nullptr : (const half *) (mask + nb33*(sequence % ne33)); ++ // opencoti F5 dca (#617B): per-sequence bases for the FRAGMENTED-fallback dense masks (same ++ // shape/stride as mask ⇒ same nb33/ne33 offset). NULL on the contiguous fast path. ++ const half * mask_succ_h = !mask_succ ? nullptr : (const half *) (mask_succ + nb33*(sequence % ne33)); ++ const half * mask_inter_h = !mask_inter ? nullptr : (const half *) (mask_inter + nb33*(sequence % ne33)); ++ float2 * dstk = ((float2 *) dst) + (sequence*int(ne01.z)*ne02 + zt_Q) * (DV/2); ++ const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); ++ ++ // opencoti F5 dca (#617B): KV_max clamps to the INTRA causal index bound — VALID only when ++ // index==pos. On the fragmented path the host passes KV_max==null (positions aren't index- ++ // ordered) so this is skipped and every phase spans the full owned KV, masks doing all selection. ++ if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } ++ const int dca_cq = pos_q[jt*ncols1] / chunk; // cq uniform per tile (analytical bands) ++ const float slope = 1.0f; // DCA: max_bias==0 ⇒ no ALiBi ++ constexpr bool is_fixup = false; ++ if (kb0_start == 0) { ++ constexpr bool needs_fixup = false; // block works on an entire tile ++ flash_attn_ext_f16_process_tile ++ (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, ++ kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); ++ } else { ++ constexpr bool needs_fixup = true; // block is missing the beginning of a tile ++ flash_attn_ext_f16_process_tile ++ (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, ++ kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); ++ } ++ ++ kbc += iter_k; ++ kbc -= kbc % iter_k; ++ kb0_start = 0; ++ kb0_stop = min(iter_k, kbc_stop - kbc); ++ } ++ ++ if (kbc >= kbc_stop) { ++ return; ++ } ++ ++ const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12); ++ const int z_KV = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa); ++ const int zt_gqa = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j); ++ const int jt = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k; ++ const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; ++ ++ const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q); ++ const float2 * Q_succ_f2 = (const float2 *) (Q_succ + nb03*sequence + nb02*zt_Q); ++ const float2 * Q_inter_f2 = (const float2 *) (Q_inter + nb03*sequence + nb02*zt_Q); ++ const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV); ++ const half * mask_h = !mask ? nullptr : (const half *) (mask + nb33*(sequence % ne33)); ++ // opencoti F5 dca (#617B): FRAGMENTED-fallback dense mask bases (tail tile). NULL on the fast path. ++ const half * mask_succ_h = !mask_succ ? nullptr : (const half *) (mask_succ + nb33*(sequence % ne33)); ++ const half * mask_inter_h = !mask_inter ? nullptr : (const half *) (mask_inter + nb33*(sequence % ne33)); ++ float2 * dstk = ((float2 *) dst) + (sequence*int(ne01.z)*ne02 + zt_Q) * (DV/2); ++ const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); ++ ++ if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } // null when fragmented ++ const int dca_cq = pos_q[jt*ncols1] / chunk; ++ const float slope = 1.0f; ++ constexpr bool is_fixup = true; // last index writes to the fixup buffer to avoid races ++ constexpr bool needs_fixup = false; ++ flash_attn_ext_f16_process_tile ++ (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, ++ ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, ++ kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); ++#else ++ GGML_UNUSED_VARS(Q, K, V, mask, mask_succ, mask_inter, sinks, KV_max, KV_min, Q_succ, Q_inter, pos_q, chunk, dst, dst_meta, ++ scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, ++ ne10, ne11, ne12, ne13, nb11, nb12, nb13, nb21, nb22, nb23, ne31, ne32, ne33, nb31, nb32, nb33); ++ NO_DEVICE_CODE; ++#endif ++} ++ ++// opencoti F5 dca perf (T87.pD Branch B + stream-k): host launch for the fused kernel. dst is the O-only ++// temp built by ggml_cuda_flash_attn_ext_lse's regime==3 branch (op_params[3]=chunk; src[4]=pos_q, ++// src[5]=Q_succ, src[6]=Q_inter). Mirrors launch_fattn: K/V already f16 (DCA), stream-k decomposition + ++// dst_tmp_meta partials + fixup (dst_lse=nullptr — fused needs no LSE out), KV_max from the INTRA mask ++// gives the per-tile causal bound. No parallel-blocks path (stream-k always covers the triangle). ++template ++void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ const ggml_tensor * PQ = dst->src[4]; // pos_q (I32) ++ const ggml_tensor * Qs = dst->src[5]; // Q_succ ++ const ggml_tensor * Qi = dst->src[6]; // Q_inter ++ const ggml_tensor * Ms = dst->src[7]; // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ fast path) ++ const ggml_tensor * Mi = dst->src[8]; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ fast path) ++ GGML_ASSERT(K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && "dca fused: f16 K/V required"); ++ GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors"); ++ // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER ++ // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent. ++ const bool fragmented = (Ms != nullptr); ++ GGML_ASSERT((Ms != nullptr) == (Mi != nullptr) && "dca fused: succ/inter masks are all-or-nothing"); ++ ++ const int id = ggml_cuda_get_device(); ++ const int cc = ggml_cuda_info().devices[id].cc; ++ constexpr int ncols = ncols1 * ncols2; ++ ++ const int nthreads = ggml_cuda_fattn_mma_get_nthreads (DKQ, DV, ncols, cc); ++ const int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa (DKQ, DV, ncols, cc); ++ const int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2 (DKQ, DV, ncols, cc); ++ const int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols, cc); ++ const int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols, cc); ++ // opencoti F5 dca (#623): MUST match the kernel's forced Q_in_reg=false on the fused path (see ++ // flash_attn_ext_f16_process_tile) so this host shared-mem sizing matches the tile_Q + separate ++ // tile_K layout the kernel uses. This is the DCA fused case function ⇒ always false. ++ const bool Q_in_reg = false; ++ // opencoti F5 dca (#623): match the kernel's single-stage clamp on the fused path (see ++ // flash_attn_ext_f16_process_tile / _iter). For DKQ 128/256 the config returns nstages_target=2 but the ++ // fused kernel forces single-stage, so size the KV smem (nbytes_shared_KV below) for 1 stage to agree. ++ const int nstages = std::min(1, ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, cc)); ++ ++ const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc)); ++ const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size; ++ const int nwarps = nthreads / warp_size_host; ++ constexpr bool V_is_K_view = DKQ == 576; ++ ++ const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(nbatch_K2 + 4, nbatch_V2 + 4) * sizeof(half2); ++ const size_t nbytes_shared_KV_2stage = nbatch_fa * (nbatch_K2 + 4 + nbatch_V2 + 4) * sizeof(half2); ++ const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2); ++ const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2); ++ const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2); ++ const size_t nbytes_shared_KV = nstages <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage; ++ const size_t nbytes_shared_total = std::max(nbytes_shared_combine, Q_in_reg ? ++ std::max(nbytes_shared_Q, nbytes_shared_KV + nbytes_shared_mask) : ++ nbytes_shared_Q + nbytes_shared_KV + nbytes_shared_mask); ++ ++ float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; ++ memcpy(&scale, (const float *) KQV->op_params + 0, sizeof(float)); ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); ++ const int chunk = ggml_get_op_params_i32(dst, 3); ++ if (logit_softcap != 0.0f) { scale /= logit_softcap; } ++ ++ ggml_cuda_pool & pool = ctx.pool(); ++ cudaStream_t main_stream = ctx.stream(); ++ ++ const int ntiles_x = (Q->ne[1] + ncols1 - 1) / ncols1; ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ const int ntiles_z_gqa = (gqa_ratio + ncols2 - 1) / ncols2; ++ const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3]; ++ ++ ggml_cuda_pool_alloc KV_max(pool); ++ ggml_cuda_pool_alloc KV_min(pool); ++ int * KV_max_ptr = nullptr; ++ // opencoti F5 dca (#617B): KV_max is the INTRA causal INDEX bound — valid only when index==pos. On ++ // the fragmented (shifted) path positions aren't index-ordered, so leave KV_max null: the kernel ++ // then spans the full KV per phase and the dense position-based masks do all selection. ++ if (!fragmented && K->ne[1] % FATTN_KQ_STRIDE == 0) { // KV_max trims INTRA to the causal bound (mask still enforces it) ++ const int s31 = mask->nb[1] / sizeof(half2); ++ const int s33 = mask->nb[3] / sizeof(half2); ++ const dim3 blk(ntiles_x, Q->ne[3], 1); ++ const dim3 bd(FATTN_KQ_STRIDE/2, 1, 1); ++ const int ne_KV_max = blk.x*blk.y; ++ const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; ++ KV_max.alloc(ne_KV_max); ++ KV_min.alloc(ne_KV_max); ++ flash_attn_mask_to_KV_max<<>> ++ ((const half2 *) mask->data, KV_max.ptr, KV_min.ptr, iter_k, s31, s33); ++ CUDA_CHECK(cudaGetLastError()); ++ KV_max_ptr = KV_max.ptr; ++ } ++ ++ const uint32_t n_head = Q->ne[2]; ++ const uint32_t n_head_log2 = 1u << uint32_t(floorf(log2f(float(n_head)))); ++ const uint3 ne01 = init_fastdiv_values(Q->ne[1]); ++ ++ // opencoti F5 dca perf (T87.pD Branch B + stream-k): load-balance the causal triangle exactly like the ++ // stock D=512 path. Replicate launch_fattn's stream-k decomposition + fixup so the fused kernel inherits ++ // it: blocks own [kb0_start,kb0_stop) of the (ntiles_dst × iter_k) work, partial (O,max,rowsum) land in ++ // dst_tmp_meta, a fixup combines seams per output tile. dst_lse=nullptr — the fused op needs no LSE out. ++ const int nsm = ggml_cuda_info().devices[id].nsm; ++ const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; ++ const dim3 block_dim(warp_size_host, nwarps, 1); ++ ggml_cuda_pool_alloc dst_tmp_meta(pool); ++ ++ auto launch = [&](auto kernel) { ++#if !defined(GGML_USE_MUSA) ++ static bool raised[GGML_CUDA_MAX_DEVICES] = {false}; ++ if (!raised[id]) { ++ CUDA_CHECK(cudaFuncSetAttribute((const void *) kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); ++ raised[id] = true; ++ } ++#endif ++ int max_blocks_per_sm = 1; ++ CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, (const void *) kernel, block_dim.x*block_dim.y*block_dim.z, nbytes_shared_total)); ++ GGML_ASSERT(max_blocks_per_sm > 0); ++ ++ // stream-k block count (verbatim from launch_fattn): round to a multiple of ntiles_dst when the ++ // occupancy loss is <=5% so each tile gets equal blocks (skips the general fixup). ++ const int max_blocks = max_blocks_per_sm*nsm; ++ const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; ++ const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); ++ const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; ++ int nblocks = ntiles_dst; ++ if (use_stream_k) { ++ const int nblocks_raw = std::min(max_blocks, ntiles_KV*ntiles_dst); ++ const int nblocks_rounded = (nblocks_raw / ntiles_dst) * ntiles_dst; ++ const int eff_loss = nblocks_rounded > 0 ? 100*(nblocks_raw - nblocks_rounded)/nblocks_raw : 100; ++ nblocks = eff_loss <= 5 ? nblocks_rounded : nblocks_raw; ++ } ++ ++ // opencoti DCA #635: decode keeps the upstream stream-k path unchanged. The chunk-boundary ++ // corruption was NOT a stream-k cross-block combine artifact — that hypothesis was falsified ++ // (decode is single-block-deterministic, and forcing stream-k decode WITH the real fix below is ++ // clean). It was a host-side INTER-Q position bug, fixed in set_input_dca (llama-kv-cache.cpp: ++ // INTER-Q = 2c, not c, so far keys land at rel-distance [c+1,2c] instead of overlapping the ++ // recent/SUCC window). Verified on the count-up reproducer in BOTH decode modes + NIAH retrieval. ++ ++ float2 * dst_meta_ptr = nullptr; ++ if (ntiles_dst % nblocks != 0) { // fixup needed only when SMs work on fractional tiles ++ dst_tmp_meta.alloc(size_t(nblocks) * ncols * (2 + DV/2)); ++ dst_meta_ptr = dst_tmp_meta.ptr; ++ } ++ ++ const dim3 grid(nblocks, 1, 1); ++ kernel<<>>( ++ (const char *) Q->data, (const char *) K->data, (const char *) V->data, ++ (const char *) mask->data, ++ Ms ? (const char *) Ms->data : nullptr, // opencoti F5 dca (#617B): dense SUCC mask (null ⇒ fast path) ++ Mi ? (const char *) Mi->data : nullptr, // opencoti F5 dca (#617B): dense INTER mask (null ⇒ fast path) ++ nullptr, KV_max_ptr, nullptr, ++ (const char *) Qs->data, (const char *) Qi->data, (const int *) PQ->data, chunk, ++ (float *) KQV->data, dst_meta_ptr, ++ scale, 0.0f /*max_bias*/, 1.0f /*m0*/, 1.0f /*m1*/, n_head_log2, logit_softcap, ++ Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], ++ K->ne[0], K->ne[1], K->ne[2], K->ne[3], K->nb[1], K->nb[2], K->nb[3], ++ V->nb[1], V->nb[2], V->nb[3], ++ mask->ne[1], mask->ne[2], mask->ne[3], mask->nb[1], mask->nb[2], mask->nb[3]); ++ CUDA_CHECK(cudaGetLastError()); ++ ++ // stream-k fixup: combine partial results across blocks for each output tile (verbatim dispatch). ++ if (nblocks % ntiles_dst == 0 && nblocks > ntiles_dst) { ++ const int bpt = nblocks / ntiles_dst; // uniform: equal blocks per tile ++ const uint3 fd0 = init_fastdiv_values(ntiles_x * ntiles_z_gqa * K->ne[2]); ++ const uint3 fd1 = init_fastdiv_values(ntiles_x * ntiles_z_gqa); ++ const uint3 fd2 = init_fastdiv_values(ntiles_x); ++ const dim3 bdc(DV, 1, 1); ++ const dim3 bnc((unsigned) ntiles_dst, ncols1, ncols2); ++ flash_attn_stream_k_fixup_uniform<<>> ++ ((float *) KQV->data, /*dst_lse=*/nullptr, dst_tmp_meta.ptr, ++ Q->ne[1], Q->ne[2], K->ne[2], nblocks, gqa_ratio, bpt, fd0, fd1, fd2); ++ CUDA_CHECK(cudaGetLastError()); ++ } else if (ntiles_dst % nblocks != 0) { ++ const int total_work = ntiles_KV * ntiles_dst; // general: fractional, unequal blocks per tile ++ const uint3 fd_k_j_z_ne12 = init_fastdiv_values(ntiles_KV * ntiles_x * ntiles_z_gqa * K->ne[2]); ++ const uint3 fd_k_j_z = init_fastdiv_values(ntiles_KV * ntiles_x * ntiles_z_gqa); ++ const uint3 fd_k_j = init_fastdiv_values(ntiles_KV * ntiles_x); ++ const uint3 fd_k = init_fastdiv_values(ntiles_KV); ++ const dim3 bdc(DV, 1, 1); ++ const dim3 bnc((unsigned) nblocks, ncols1, ncols2); ++ flash_attn_stream_k_fixup_general<<>> ++ ((float *) KQV->data, /*dst_lse=*/nullptr, dst_tmp_meta.ptr, ++ Q->ne[1], Q->ne[2], gqa_ratio, total_work, fd_k_j_z_ne12, fd_k_j_z, fd_k_j, fd_k); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++ }; ++ if (logit_softcap == 0.0f) { ++ launch(flash_attn_ext_f16_dca_fused); ++ } else { ++ launch(flash_attn_ext_f16_dca_fused); ++ } ++} ++ + #define DECL_FATTN_MMA_F16_CASE(DKQ, DV, ncols1, ncols2) \ + template void ggml_cuda_flash_attn_ext_mma_f16_case \ + (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh +@@ -794,6 +794,7 @@ static __global__ void flash_attn_tile( + const char * __restrict__ mask, + const char * __restrict__ sinks, + const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -819,7 +820,7 @@ static __global__ void flash_attn_tile( + #endif // GGML_USE_WMMA_FATTN + (use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512)) + ) { +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -946,17 +947,22 @@ static __global__ void flash_attn_tile( + + // Main loop over KV cache: + const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; ++ // opencoti F5 dca perf (T87.pD-opt O4): low band edge — skip K/V tiles entirely below it (DCA ++ // INTRA/SUCC regimes, SWA windows). k_VKQ_min==0 for causal masks → the guards below never fire. ++ const int k_VKQ_min = KV_min ? KV_min[sequence*gridDim.x + blockIdx.x] : 0; + if (ncols2 == 1) { + // Branch with out-of-bounds checks. + int k_VKQ_0 = blockIdx.y*nbatch_fa; + while (k_VKQ_0 < k_VKQ_max - nbatch_fa) { +- constexpr bool oob_check = false; +- flash_attn_tile_iter +- (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, +- stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); ++ if (k_VKQ_0 + nbatch_fa > k_VKQ_min) { ++ constexpr bool oob_check = false; ++ flash_attn_tile_iter ++ (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, ++ stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); ++ } + k_VKQ_0 += gridDim.y*nbatch_fa; + } +- if (k_VKQ_0 < k_VKQ_max) { ++ if (k_VKQ_0 < k_VKQ_max && k_VKQ_0 + nbatch_fa > k_VKQ_min) { + constexpr bool oob_check = true; + flash_attn_tile_iter + (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, +@@ -965,6 +971,9 @@ static __global__ void flash_attn_tile( + } else { + // Branch without out-of-bounds checks. + for (int k_VKQ_0 = blockIdx.y*nbatch_fa; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nbatch_fa) { ++ if (k_VKQ_0 + nbatch_fa <= k_VKQ_min) { ++ continue; ++ } + constexpr bool oob_check = false; + flash_attn_tile_iter + (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, +@@ -1126,7 +1135,7 @@ static __global__ void flash_attn_tile( + } + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -25,6 +25,7 @@ static __global__ void flash_attn_ext_vec( + const char * __restrict__ mask, + const char * __restrict__ sinks, + const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -45,7 +46,7 @@ static __global__ void flash_attn_ext_vec( + + // Skip unused kernel variants for faster compilation: + if (use_logit_softcap && !(D == 128 || D == 256)) { +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -246,12 +247,19 @@ static __global__ void flash_attn_ext_vec( + } + + const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; ++ const int k_VKQ_min = KV_min ? KV_min[sequence*gridDim.x + blockIdx.x] : 0; // opencoti F5 dca perf (T87.pD-opt O4) + K += blockIdx.y*nthreads * nb11; + V += blockIdx.y*nthreads * nb21; + maskh += blockIdx.y*nthreads; + for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads, + // Increment pointers after each loop: + K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) { ++ // opencoti F5 dca perf (T87.pD-opt O4): skip K/V tiles entirely below the low band edge ++ // (DCA INTRA/SUCC regimes, SWA windows). k_VKQ_min==0 for causal masks → never skips. The ++ // for-increment above still advances the K/V/mask pointers, so they stay in sync. ++ if (k_VKQ_0 + nthreads <= k_VKQ_min) { ++ continue; ++ } + + // Calculate KQ tile and keep track of new maximum KQ values: + float KQ_reg[ncols]; // KQ in registers. +@@ -512,7 +520,7 @@ static __global__ void flash_attn_ext_vec( + dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]); + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +@@ -30,6 +30,7 @@ static __global__ void flash_attn_ext_f16( + const char * __restrict__ mask, + const char * __restrict__ sinks, + const int * __restrict__ KV_max, ++ const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -190,7 +191,13 @@ static __global__ void flash_attn_ext_f16( + + // Iterate over ne11 == previous tokens: + const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; ++ const int k_VKQ_min = KV_min ? KV_min[sequence*gridDim.x + blockIdx.x] : 0; // opencoti F5 dca perf (T87.pD-opt O4) + for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) { ++ // opencoti F5 dca perf (T87.pD-opt O4): skip K/V tiles entirely below the low band edge ++ // (DCA INTRA/SUCC regimes, SWA windows). k_VKQ_min==0 for causal masks → never skips. ++ if (k_VKQ_0 + FATTN_KQ_STRIDE <= k_VKQ_min) { ++ continue; ++ } + // Calculate tile of KQ: + #pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) { +@@ -494,7 +501,7 @@ static __global__ void flash_attn_ext_f16( + dst_meta[j_dst_unrolled] = dst_meta_val; + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -10,6 +10,23 @@ + // declared in fattn-common.cuh. Set immediately before a single launch_fattn dispatch to request a + // per-row log-sum-exp output; launch_fattn drains it back to nullptr. See docs/protocols/UPSTREAM_SYNC.md. + thread_local float * opencoti_fattn_dst_lse = nullptr; ++// opencoti F5 dca perf (T87.pD-opt O2): set by launch_fattn at its dispatch site to report whether a ++// finalize kernel (the populator of dst_lse) actually ran. The DCA _lse op reads it right after the FA ++// call to decide whether the LSE was emitted for free (common: large-KV prefill/decode takes a finalize ++// path) or must be recomputed by the streaming_lse_kernel fallback (rare: in-kernel single-block path). ++thread_local bool opencoti_fattn_dst_lse_written = false; ++ ++// opencoti F5 dca perf (T87.pD-opt O6 Tier B): analytical band-bounds side-channel. A DCA SUCC/INTER ++// regime is a CONTIGUOUS causal key-band per query ([(cq-1)c, cq*c) for SUCC, [0,(cq-1)c) for INTER, ++// cq = pos_q/c) that needs NO per-element causal — every band key is < pos_q for every query. So instead ++// of building+uploading a dense [n_kv x n_tokens] mask (the host O(n_kv*n_tokens) prefill bottleneck), ++// ggml_cuda_flash_attn_ext_lse sets these before the inner FA dispatch; launch_fattn then fills the ++// per-tile KV_min/KV_max ANALYTICALLY from pos_q (device, len n_q) + chunk and passes mask=nullptr. ++// Drained back to nullptr/0/-1 after the dispatch (mirrors opencoti_fattn_dst_lse). regime is ++// DCA_REGIME_SUCC(1) / _INTER(2); INTRA(0) keeps its native-causal mask and never sets these. ++thread_local const int * opencoti_fattn_dca_pos_q = nullptr; ++thread_local int opencoti_fattn_dca_chunk = 0; ++thread_local int opencoti_fattn_dca_regime = -1; + + template + static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +@@ -375,7 +392,14 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + + // The effective batch size for the kernel can be increased by gqa_ratio. + // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded, +- bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B / Path B): SUCC/INTER analytical-band regimes pass mask=nullptr ++ // and mask purely via the on-GPU KV band (kb0_min/kb0_stop from KV_min/KV_max). ggml_cuda_flash_attn_ext_lse ++ // arms the thread_local opencoti_fattn_dca_pos_q around the inner FA call, so it is readable here at selection ++ // time → allow the gqa-batched MMA kernel for a null-mask band op. The band-aware D512 kernel (fattn-mma-f16.cuh) ++ // skips the mask load/add when mask_h==nullptr and relies solely on the tile band. In every reachable STOCK ++ // state ncols2>1 ⟹ mask!=null, so this disjunct changes nothing outside DCA. ++ const bool dca_band = (opencoti_fattn_dca_pos_q != nullptr); ++ bool gqa_opt_applies = gqa_ratio >= 2 && (mask || dca_band) && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + for (const ggml_tensor * t : {Q, K, V, mask}) { + if (t == nullptr || ggml_is_quantized(t->type)) { + continue; +@@ -597,6 +621,192 @@ bool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst) { + return ggml_cuda_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE; + } + ++// opencoti F5 dca (#593) — fwd-decl: streaming_lse_kernel is defined ~line 645, but the LSE-op ++// forward below launches it from above its definition; declare it here so the call resolves. ++static __global__ void streaming_lse_kernel( ++ const char * __restrict__ Q, const char * __restrict__ K, ++ const char * __restrict__ mask, float * __restrict__ lse, ++ const float scale, const float logit_softcap, const int tile_kv, const int gqa_ratio, ++ const int head_dim, const int n_head, const int n_q, ++ const int64_t q_nb1, const int64_t q_nb2, const int64_t q_nb3, ++ const int64_t k_nb1, const int64_t k_nb2, const int64_t k_nb3, ++ const int64_t m_nb0, const int64_t m_nb1, ++ // opencoti F5 dca perf (O6): analytical band for SUCC/INTER (defaults keep legacy 18-arg callers). ++ const int * __restrict__ dca_pos_q = nullptr, const int dca_chunk = 0, const int dca_regime = 0); ++ ++// opencoti F5 dca (#593) — CUDA forward for GGML_OP_FLASH_ATTN_EXT_LSE. Runs a standard ++// flash-attention for the NORMALIZED O (written into the leading DV*n_rows of dst, shaped ++// exactly like flash_attn_ext) via the proven streaming-op trick of an O-typed view of dst; ++// then computes the per-row LSE into the trailing n_rows of dst with streaming_lse_kernel — ++// the same softcap-aware recompute the rolling-KV streaming op uses for its no-finalize ++// path, so it works for ANY n_q (incl. 512k prefill) and avoids the decode_lse / bug-263 ++// finalize fragility. dst packs [O (DV*n_rows) | lse (n_rows)] (ne[0] = DV+1). The DCA ++// 3-regime graph-math combine reads O and lse back via offset views. ++// opencoti F5 dca perf (T87.pD Branch B): ncols1/ncols2 selection for the FUSED DCA kernel. DKQ=DV is the ++// model head dim (Gemma-4 26B-A4B = 512, qwen35moe = 256, qwen2/qwen3/qwen3moe = 128). Mirrors the stock ++// switch_ncols2/switch_ncols1 GQA-opt path (the fused op guarantees a mask, max_bias==0, aligned f16 K/V ⇒ ++// use_gqa_opt is always true here). ++// opencoti F5 dca (#623): templated on DKQ so each arch gets a head-dim-correct kernel. The earlier code ++// hardcoded <512,512> (Gemma's head dim) for EVERY model: supports_op (ggml-cuda.cu) admits head_dim ++// 128/256/512, so Qwen ops reached this fused path and launched a 512-wide kernel over 256/128-wide K/V/Q — ++// striding past each head into the next head's memory → multi-chunk garbage from token 1 (no crash). The ++// fused case + process_tile are already DKQ-templated (softcap whitelist DKQ∈{128,256,512}); the config ++// table has matching entries for all three. The single_chunk path (plain ggml_flash_attn_ext) was never ++// affected — only the analytical-bands fused dispatch went through here. ++template ++static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ if constexpr (ncols2 <= 8) { ++ if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ } ++ } ++ if constexpr (ncols2 <= 16) { ++ if ((turing_mma_available(cc) || amd_wmma_available(cc)) && Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ } ++ } ++ if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || amd_wmma_available(cc) || Q->ne[1] <= 32/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); ++} ++ ++// gqa_ratio → ncols2 selection for a fixed head dim DKQ (the old switch body, now DKQ-parameterized). ++template ++static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ if (cc == GGML_CUDA_CC_VOLTA) { ++ if (gqa_ratio % 8 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio % 4 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio % 2 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; ++ } ++ if (gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio > 2) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio > 1) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); ++} ++ ++// opencoti F5 dca (#623): head-dim switch mirroring the stock ggml_cuda_flash_attn_ext_mma_f16 path. DCA ++// only ever uses DKQ==DV (key_length==value_length on every supported arch), so dispatch on Q->ne[0] alone. ++static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * V = dst->src[2]; ++ switch (Q->ne[0]) { ++ case 128: ++ GGML_ASSERT(V->ne[0] == 128); ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<128>(ctx, dst); break; ++ case 256: ++ GGML_ASSERT(V->ne[0] == 256); ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256>(ctx, dst); break; ++ case 512: ++ GGML_ASSERT(V->ne[0] == 512); ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512>(ctx, dst); break; ++ default: ++ GGML_ABORT("dca fused: unsupported head dim %d (expected 128/256/512)", (int) Q->ne[0]); ++ break; ++ } ++} ++ ++void ggml_cuda_flash_attn_ext_lse(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ ggml_cuda_set_device(ctx.device); ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * M = dst->src[3]; ++ ++ const int64_t DV = V->ne[0]; ++ const int64_t n_head = Q->ne[2]; ++ const int64_t n_q = Q->ne[1]; ++ const int64_t n_batch = Q->ne[3]; ++ const int64_t n_rows = n_head * n_q * n_batch; ++ GGML_ASSERT(dst->ne[0] == DV + 1 && "flash_attn_ext_lse dst must pack [O|lse] on ne[0]"); ++ GGML_ASSERT(Q->type == GGML_TYPE_F32 && "flash_attn_ext_lse expects f32 Q for the lse recompute"); ++ GGML_ASSERT(K->type == GGML_TYPE_F16 && "flash_attn_ext_lse expects f16 K for the lse recompute"); ++ ++ // (1) NORMALIZED O into the leading DV*n_rows of dst (contiguous [DV,n_head,n_q,n_batch]). ++ ggml_tensor o = *dst; ++ o.op = GGML_OP_FLASH_ATTN_EXT; ++ o.src[4] = nullptr; // no attention sinks for DCA ++ // opencoti F5 dca perf (O6): the DCA band metadata travels via the side-channel, not op_params; clear ++ // op_params[3..4] on the inner EXT temp so no downstream op_params reader can misinterpret them. ++ o.op_params[3] = 0; ++ o.op_params[4] = 0; ++ o.ne[0] = DV; o.ne[1] = n_head; o.ne[2] = n_q; o.ne[3] = n_batch; ++ o.nb[0] = sizeof(float); ++ o.nb[1] = o.nb[0] * o.ne[0]; ++ o.nb[2] = o.nb[1] * o.ne[1]; ++ o.nb[3] = o.nb[2] * o.ne[2]; ++ o.data = dst->data; ++ ++ // opencoti F5 dca perf (T87.pD-opt O2): the FA finalize kernels already compute the per-row LSE ++ // (max + log(sumexp)) and can write it for free via the rolling-kv side-channel. Point it at the ++ // SAME trailing region streaming_lse_kernel would fill — the finalize row order (h + n_head*qi + ++ // n_head*n_q*b) matches O's row order, which is exactly what the host 3-regime combine reads back. ++ // launch_fattn sets opencoti_fattn_dst_lse_written iff a finalize path ran; on the rare in-kernel ++ // single-block path it stays false and we fall back to the streaming recompute below. This removes ++ // one full O(L^2) pass per regime on the multi-chunk (>chunk) path. ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): when this is a SUCC/INTER regime (src[4]=pos_q set, ++ // mask=src[3]=nullptr), arm the analytical band side-channel so launch_fattn fills KV_min/KV_max from ++ // positions (no dense mask to scan). INTRA leaves src[4]=NULL ⇒ side-channel disarmed (legacy path). ++ const ggml_tensor * PQ = dst->src[4]; ++ const int dca_chunk = ggml_get_op_params_i32(dst, 3); ++ const int dca_regime = ggml_get_op_params_i32(dst, 4); ++ ++ // opencoti F5 dca perf (T87.pD Branch B): FUSED single-pass DCA. The three band passes (INTER/SUCC/ ++ // INTRA) are folded into ONE block-per-tile kernel that streams K/V once and swaps the pre-roped Q ++ // variant at the 2 phase boundaries — no per-regime LSE emit, no host combine, no streaming_lse ++ // fallback. The fused case reads its inputs straight off dst (src[0]=Q_intra, src[1]=K, src[2]=V, ++ // src[3]=mask_intra, src[4]=pos_q_real, src[5]=Q_succ, src[6]=Q_inter, op_params[3]=chunk) and writes ++ // normalized O into the leading DV*n_rows of dst's [O|lse] buffer (the trailing lse row is left ++ // untouched — dca.cpp slices it off). Bands partition the causal range [0,i] ⇒ one online-softmax ++ // pass is bit-faithful to the 3-op + host LSE combine it replaces. ++ if (dca_regime == 3) { ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst); ++ return; ++ } ++ ++ float * lse_out = (float *) dst->data + DV * n_rows; ++ opencoti_fattn_dst_lse = lse_out; ++ opencoti_fattn_dst_lse_written = false; ++ if (PQ) { ++ opencoti_fattn_dca_pos_q = (const int *) PQ->data; ++ opencoti_fattn_dca_chunk = dca_chunk; ++ opencoti_fattn_dca_regime = dca_regime; ++ } ++ ggml_cuda_flash_attn_ext(ctx, &o); ++ opencoti_fattn_dst_lse = nullptr; // defensive: don't leak the side-channels into the next FA op ++ opencoti_fattn_dca_pos_q = nullptr; ++ opencoti_fattn_dca_chunk = 0; ++ opencoti_fattn_dca_regime = -1; ++ ++ // (2) Fallback per-row LSE recompute — only when the finalize didn't emit it for free. ++ if (!opencoti_fattn_dst_lse_written) { ++ float scale = 1.0f, logit_softcap = 0.0f; ++ memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); ++ memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); ++ const int gqa_ratio = (int) (n_head / K->ne[2]); ++ cudaStream_t stream = ctx.stream(); ++ streaming_lse_kernel<<<(unsigned) n_rows, WARP_SIZE, 0, stream>>>( ++ (const char *) Q->data, (const char *) K->data, ++ M ? (const char *) M->data : nullptr, lse_out, ++ scale, logit_softcap, (int) K->ne[1], gqa_ratio, (int) K->ne[0], ++ (int) n_head, (int) n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], ++ M ? M->nb[0] : 0, M ? M->nb[1] : 0, ++ // opencoti F5 dca perf (O6): band-aware LSE for SUCC/INTER when the FA took a no-finalize ++ // path (rare; e.g. decode n_q==1). PQ==NULL (INTRA/legacy) ⇒ regime 0 ⇒ full-KV scan. ++ PQ ? (const int *) PQ->data : nullptr, dca_chunk, PQ ? dca_regime : 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } ++} ++ + // ============================================================================ + // opencoti-hook: f5-rolling-kv — W4 M7-D #304 / R2-b S2 (#311). + // Streaming flash-attention via key-axis tiling + LSE-weighted online-softmax. +@@ -636,7 +846,13 @@ static __global__ void streaming_lse_kernel( + const int head_dim, const int n_head, const int n_q, + const int64_t q_nb1, const int64_t q_nb2, const int64_t q_nb3, + const int64_t k_nb1, const int64_t k_nb2, const int64_t k_nb3, +- const int64_t m_nb0, const int64_t m_nb1) { ++ const int64_t m_nb0, const int64_t m_nb1, ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): analytical band for SUCC/INTER. When dca_regime>0 ++ // there is no mask; this fallback (rare: no-finalize path, e.g. decode n_q==1) must compute the ++ // LSE over EXACTLY the regime's contiguous key band [lo,hi) so it matches the band-limited O the ++ // FA produced (KV_min/KV_max). Key index == position (single-stream contiguous prefill). ++ // (defaults are declared on the forward declaration above — C++ forbids repeating them here.) ++ const int * __restrict__ dca_pos_q, const int dca_chunk, const int dca_regime) { + const int row = blockIdx.x; // flattened (h, qi, b) + const int h = row % n_head; + const int qi = (row / n_head) % n_q; +@@ -648,9 +864,22 @@ static __global__ void streaming_lse_kernel( + const char * Kb = K + (int64_t)hk*k_nb2 + (int64_t)b*k_nb3; + const char * Mb = mask ? (mask + (int64_t)qi*m_nb1) : nullptr; + ++ // opencoti F5 dca O6: derive this query's regime band [band_lo, band_hi) in key indices. ++ int band_lo = 0, band_hi = tile_kv; ++ if (dca_regime > 0) { ++ const int p1 = dca_pos_q[qi]; ++ const int cq = p1 / dca_chunk; ++ if (dca_regime == 1) { band_lo = (cq - 1) * dca_chunk; band_hi = cq * dca_chunk; } // SUCC ++ else { band_lo = 0; band_hi = (cq - 1) * dca_chunk; } // INTER ++ if (band_lo < 0) { band_lo = 0; } ++ if (band_hi < 0) { band_hi = 0; } ++ if (band_hi > tile_kv) { band_hi = tile_kv; } ++ if (band_lo > band_hi) { band_lo = band_hi; } ++ } ++ + float m = -INFINITY; + float l = 0.0f; +- for (int j = 0; j < tile_kv; ++j) { ++ for (int j = band_lo; j < band_hi; ++j) { + const half * Kj = (const half *)(Kb + (int64_t)j*k_nb1); + float part = 0.0f; + for (int d = lane; d < head_dim; d += WARP_SIZE) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +@@ -4,6 +4,9 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst + + bool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst); + ++// opencoti F5 dca — #593: flash-attention emitting normalized O + per-row lse (packed [DV+1]). ++void ggml_cuda_flash_attn_ext_lse(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++ + // opencoti-hook: f5-rolling-kv — W4 M7-D #304 / R2-b S2 (#311). + // Streaming flash-attention: key-axis tiling + LSE-weighted online-softmax + // combine over the stock FA kernels (which stay untouched). Falls back to +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -3146,6 +3146,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg + case GGML_OP_FLASH_ATTN_EXT: + ggml_cuda_flash_attn_ext(ctx, dst); + break; ++ case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 ++ ggml_cuda_flash_attn_ext_lse(ctx, dst); ++ break; + case GGML_OP_CROSS_ENTROPY_LOSS: + ggml_cuda_cross_entropy_loss(ctx, dst); + break; +@@ -5615,6 +5618,23 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + #endif // GGML_USE_MUSA + case GGML_OP_FLASH_ATTN_EXT: + return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op); ++ case GGML_OP_FLASH_ATTN_EXT_LSE: { // opencoti F5 dca — #593 ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B / Path B): a SUCC/INTER analytical-band regime carries ++ // mask=src[3]=null and pos_q=src[4]!=null. It masks purely via the on-GPU KV band; the band-aware D512 ++ // MMA kernel handles a null mask in the gqa-batched path. The strict FA-supported check (below) rejects ++ // a null mask at D512, so short-circuit to supported here. The op is CUDA-only (no CPU impl) and its ++ // Q/K/V shapes are identical to the INTRA regime's op which strict-passes, so dims are valid by ++ // construction. INTRA keeps src[3]=mask, src[4]=null → falls through to the strict path below. ++ if (op->src[3] == nullptr && op->src[4] != nullptr) { ++ return true; ++ } ++ ggml_tensor o = *op; // dst packs [O|lse] (ne[0]=DV+1); check the O FA (ne[0]=DV) ++ o.ne[0] = op->src[2]->ne[0]; ++ o.src[4] = nullptr; // opencoti F5 dca perf (O6): src[4] is DCA pos_q (I32), NOT attention ++ // sinks — clear it so the FA-supported check (which reads src[4] as ++ // sinks) doesn't see a non-F32 tensor and report unsupported→CPU. ++ return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, &o); ++ } + case GGML_OP_STREAMING_FLASH_ATTN: { + // opencoti-hook: f5-rolling-kv — bug-259. The streaming forward repacks + // its (possibly non-contiguous / pinned-host) K/V into CONTIGUOUS f16 +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -1131,9 +1131,10 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { + "STREAMING_FLASH_ATTN", + "FLASH_ATTN_TAIL_PARTIAL", + "TURBO_WHT", ++ "FLASH_ATTN_EXT_LSE", + }; + +-static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); ++static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); + + static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "none", +@@ -1245,9 +1246,10 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "streaming_flash_attn(q,k,v)", + "flash_attn_tail_partial(q,k,v)", + "turbo_wht(x)", ++ "flash_attn_ext_lse(q,k,v)", + }; + +-static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100"); ++static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); + + static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); + +@@ -5628,6 +5630,137 @@ struct ggml_tensor * ggml_flash_attn_ext_tail_partial( + return result; + } + ++// opencoti F5 dca (#593) — flash-attention emitting NORMALIZED O + per-row LSE packed on ++// the head-dim axis: dst = [DV+1, n_head, n_q, n_batch]; row [0,DV) = O, row [DV] = lse ++// (= m + logf(sum exp); softcap + sinks folded in by the kernel). Same construction as ++// ggml_flash_attn_ext but the CUDA forward also writes the per-row LSE the DCA 3-regime ++// graph-math combine consumes (O = sum_r w_r O_r / sum_r w_r, w_r = exp(lse_r - max)). ++struct ggml_tensor * ggml_flash_attn_ext_lse( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask, ++ float scale, ++ float max_bias, ++ float logit_softcap, ++ struct ggml_tensor * pos_q, ++ int chunk_size, ++ int regime) { ++ GGML_ASSERT(ggml_can_mul_mat(k, q)); ++ GGML_ASSERT(q->ne[3] == k->ne[3]); ++ GGML_ASSERT(q->ne[3] == v->ne[3]); ++ if (mask) { ++ GGML_ASSERT(mask->type == GGML_TYPE_F16); ++ GGML_ASSERT(ggml_is_contiguous(mask)); ++ GGML_ASSERT(q->ne[2] % mask->ne[2] == 0); ++ GGML_ASSERT(q->ne[3] % mask->ne[3] == 0); ++ } ++ if (max_bias > 0.0f) { ++ GGML_ASSERT(mask); ++ } ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): analytical band-bounds form for SUCC/INTER. ++ // pos_q is the device I32 query-position vector (len n_q); the CUDA op derives the per-tile KV band ++ // from pos_q[0]/chunk_size for the regime, so NO dense mask is needed (and max_bias must be 0 — ++ // ALiBi would need the mask). INTRA passes pos_q==NULL and keeps its native-causal mask. ++ if (pos_q) { ++ GGML_ASSERT(mask == NULL && "dca O6: analytical (pos_q) regimes pass mask=NULL"); ++ GGML_ASSERT(pos_q->type == GGML_TYPE_I32); ++ GGML_ASSERT(pos_q->ne[0] == q->ne[1] && "dca O6: pos_q length must equal n_q"); ++ GGML_ASSERT(max_bias == 0.0f && "dca O6: analytical band path does not support ALiBi"); ++ GGML_ASSERT(chunk_size > 0 && (regime == 1 || regime == 2) && "dca O6: regime is SUCC(1)/INTER(2)"); ++ } ++ ++ // permute(0,2,1,3) output like flash_attn_ext, with ONE extra head-dim slot for lse. ++ int64_t ne[4] = { v->ne[0] + 1, q->ne[2], q->ne[1], q->ne[3] }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ float params[] = { scale, max_bias, logit_softcap }; ++ ggml_set_op_params(result, params, sizeof(params)); ++ // op_params[3..4]: DCA analytical band metadata (0 for the legacy INTRA/mask form; ggml_new_tensor ++ // zero-inits op_params, so a pre-O6 call without pos_q leaves these at 0 ⇒ CUDA takes the mask path). ++ ggml_set_op_params_i32(result, 3, pos_q ? chunk_size : 0); ++ ggml_set_op_params_i32(result, 4, pos_q ? regime : 0); ++ ++ result->op = GGML_OP_FLASH_ATTN_EXT_LSE; ++ result->src[0] = q; ++ result->src[1] = k; ++ result->src[2] = v; ++ result->src[3] = mask; ++ result->src[4] = pos_q; // opencoti F5 dca perf (O6): device I32 query positions (NULL for INTRA) ++ ++ return result; ++} ++ ++// opencoti F5 dca perf (T87.pD Branch B) — FUSED single-pass DCA flash-attention. Encoded as a ++// GGML_OP_FLASH_ATTN_EXT_LSE node with regime=3 (FUSED); the CUDA forward walks the causal range in ++// 3 contiguous key-phases (INTER→SUCC→INTRA), reloading the matching pre-roped Q at each phase ++// boundary, with one continuous online-softmax — bit-faithful to the 3-op + host-combine path (the ++// regime bands partition [0,i]). Output keeps the [DV+1,...] LSE layout for table reuse; the lse row ++// is unused (the single pass needs no combine). ++struct ggml_tensor * ggml_flash_attn_ext_dca_fused( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q_intra, ++ struct ggml_tensor * q_succ, ++ struct ggml_tensor * q_inter, ++ struct ggml_tensor * k, ++ struct ggml_tensor * v, ++ struct ggml_tensor * mask_intra, ++ struct ggml_tensor * mask_succ, ++ struct ggml_tensor * mask_inter, ++ struct ggml_tensor * pos_q_real, ++ float scale, ++ float max_bias, ++ float logit_softcap, ++ int chunk_size) { ++ GGML_ASSERT(ggml_can_mul_mat(k, q_intra)); ++ GGML_ASSERT(q_intra->ne[3] == k->ne[3]); ++ GGML_ASSERT(q_intra->ne[3] == v->ne[3]); ++ // the 3 Q variants are the same query tensor re-roped per regime: identical shape. ++ GGML_ASSERT(q_succ && ggml_are_same_shape(q_intra, q_succ)); ++ GGML_ASSERT(q_inter && ggml_are_same_shape(q_intra, q_inter)); ++ GGML_ASSERT(mask_intra && mask_intra->type == GGML_TYPE_F16); ++ GGML_ASSERT(ggml_is_contiguous(mask_intra)); ++ GGML_ASSERT(q_intra->ne[2] % mask_intra->ne[2] == 0); ++ GGML_ASSERT(q_intra->ne[3] % mask_intra->ne[3] == 0); ++ // opencoti F5 dca (#617B): FRAGMENTED-fallback masks travel together (both NULL on the contiguous ++ // fast path, both present on a shifted cache). When present they must match the INTRA mask's dtype ++ // + shape so the kernel can offset all three identically (same stride_mask). ++ GGML_ASSERT((mask_succ == NULL) == (mask_inter == NULL) && "dca fused: succ/inter masks are all-or-nothing"); ++ if (mask_succ) { ++ GGML_ASSERT(mask_succ->type == GGML_TYPE_F16 && ggml_is_contiguous(mask_succ)); ++ GGML_ASSERT(mask_inter->type == GGML_TYPE_F16 && ggml_is_contiguous(mask_inter)); ++ GGML_ASSERT(ggml_are_same_shape(mask_intra, mask_succ)); ++ GGML_ASSERT(ggml_are_same_shape(mask_intra, mask_inter)); ++ } ++ GGML_ASSERT(pos_q_real && pos_q_real->type == GGML_TYPE_I32); ++ GGML_ASSERT(pos_q_real->ne[0] == q_intra->ne[1] && "dca fused: pos_q_real length must equal n_q"); ++ GGML_ASSERT(max_bias == 0.0f && "dca fused: analytical band path does not support ALiBi"); ++ GGML_ASSERT(chunk_size > 0); ++ ++ // same [DV+1, n_head, n_q, n_batch] layout as ggml_flash_attn_ext_lse (lse row unused here). ++ int64_t ne[4] = { v->ne[0] + 1, q_intra->ne[2], q_intra->ne[1], q_intra->ne[3] }; ++ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); ++ ++ float params[] = { scale, max_bias, logit_softcap }; ++ ggml_set_op_params(result, params, sizeof(params)); ++ ggml_set_op_params_i32(result, 3, chunk_size); ++ ggml_set_op_params_i32(result, 4, 3); // regime==3 ⇒ FUSED (CUDA forward branches on this) ++ ++ result->op = GGML_OP_FLASH_ATTN_EXT_LSE; ++ result->src[0] = q_intra; ++ result->src[1] = k; ++ result->src[2] = v; ++ result->src[3] = mask_intra; ++ result->src[4] = pos_q_real; // device I32 real abs query positions (band channel) ++ result->src[5] = q_succ; // pre-roped Q for the SUCC phase ++ result->src[6] = q_inter; // pre-roped Q for the INTER phase ++ result->src[7] = mask_succ; // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ analytical fast path) ++ result->src[8] = mask_inter; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ analytical fast path) ++ ++ return result; ++} ++ + // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391. Walsh–Hadamard + // rotation op for the fused turbo FA-VEC path (Q forward-rotate / FA-output + // inverse-rotate). B-adapted vs the AtomicBot fork: `scale` is normally NULL and +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -376,6 +376,11 @@ extern "C" { + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md + // true = fuse decode MoE up+gate+GLU into one op. false = off (default). + bool fused_moe_up_gate; ++ // opencoti F5 dca — Dual Chunk Attention training-free long-context (docs/features/gemma4_dca.md) ++ // dca_enabled: false = off (default). dca_chunk_size: 0 = auto. dca_yarn_factor: 1.0 = none. ++ bool dca_enabled; ++ uint32_t dca_chunk_size; ++ float dca_yarn_factor; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/src/dca.cpp +@@ -0,0 +1,327 @@ ++#include "dca.h" ++ ++#include "ggml.h" ++#include "llama-graph.h" ++#include "llama-kv-cache.h" ++#include "llama-kv-cache-iswa.h" ++ ++#include ++#include ++#include ++ ++// opencoti F5 dca (#593, T87.pD) — Section C host-fill + input builder. Design + rationale: ++// docs/features/gemma4_dca.md (Section C). This file lands the DCA 3-regime position-remap INPUT ++// (built once per ubatch). The build_attn_dca graph bodies that consume it rope the small Q per ++// regime and run the fused single-pass analytical-band FA over the PRE-roped cached K (#617: K is ++// roped once at insert, not here; #618: the legacy 3x flash_attn_ext_lse + host LSE-combine fallback ++// was removed). The host-side per-cell fill lives on the kv-cache (set_input_dca) because only it can ++// read v_cells positions; set_input() delegates. ++ ++uint32_t dca_resolve_chunk_size(const llama_cparams & cparams, const llama_hparams & hparams) { ++ if (cparams.dca_chunk_size != 0) { ++ return cparams.dca_chunk_size; ++ } ++ // auto: the model's original/pretrain window (so no intra/succ q-k distance exceeds it). ++ uint32_t c = cparams.n_ctx_orig_yarn; ++ if (c == 0) { ++ c = hparams.n_ctx_train; ++ } ++ GGML_ASSERT(c >= 1 && "dca: could not resolve a positive chunk size"); ++ return c; ++} ++ ++// opencoti F5 dca (#623): qwen35/qwen35moe carry the MROPE bit (rope_type IMROPE=40); the DCA per-regime ++// position tensors are then 4-wide (section split) and rope via ggml_rope_multi. NEOX/NORMAL archs are ++// 1-wide via ggml_rope_ext. See the build_dca_rope rationale block below. ++static inline bool dca_is_mrope(int rope_type) { ++ return (rope_type & GGML_ROPE_TYPE_MROPE) != 0; ++} ++// Position ids per token the DCA input must allocate/fill: 4 (MROPE section split) or 1 (NEOX scalar). ++static inline int dca_pos_per_token(int rope_type) { ++ return dca_is_mrope(rope_type) ? GGML_MROPE_SECTIONS : 1; ++} ++ ++void llm_graph_input_dca::set_input(const llama_ubatch * ubatch) { ++ if (!mctx) { ++ return; ++ } ++ // Delegate to the kv-cache: it owns v_cells, the only source of per-cell cached positions. ++ mctx->set_input_dca( ++ pos_k, ++ pos_q[DCA_REGIME_INTRA], pos_q[DCA_REGIME_SUCC], pos_q[DCA_REGIME_INTER], ++ mask[DCA_REGIME_INTRA], mask[DCA_REGIME_SUCC], mask[DCA_REGIME_INTER], ++ pos_q_real, // O6-fix: real abs query positions for the analytical band (null when not analytical) ++ ubatch, chunk_size); ++} ++ ++bool llm_graph_input_dca::can_reuse(const llm_graph_params & params) { ++ // pos_k length tracks the (growing) KV count and the per-regime masks are position-specific, so ++ // the DCA input is rebuilt every ubatch (conservative; set_input re-fills regardless). A future ++ // optimization could reuse when n_kv + n_tokens shapes match, like can_reuse_kq_mask. ++ (void) params; ++ return false; ++} ++ ++// Allocate the DCA input tensors and register the input. mctx_kv = the BASE kv-cache context: for ++// Gemma-4 globals that is build_attn_inp_kv_iswa()->mctx->get_base(); for Qwen it is the plain kv ++// context. n_kv / n_tokens follow build_attn_inp_kq_mask (single-stream; DCA asserts n_stream==1). ++llm_graph_input_dca * llm_graph_context::build_attn_inp_dca(const llama_kv_cache_context * mctx_kv) const { ++ const uint32_t c = dca_resolve_chunk_size(cparams, hparams); ++ ++ auto inp = std::make_unique(hparams, cparams, mctx_kv, c); ++ ++ const int64_t n_kv = mctx_kv->get_n_kv(); // used for the per-regime mask shapes below ++ ++ // opencoti F5 dca (#617B): is this ubatch's cache non-contiguous (cell index != absolute position, ++ // post context-shift / self-extend)? Read the STICKY latch at graph-build time. When true on the ++ // multi-chunk (analytical) path we additionally allocate the dense SUCC/INTER fallback masks so the ++ // host fill + fused kernel can select keys by actual position instead of the (now-invalid) index ++ // bands. ALWAYS false on the hot path (prefill, decode within n_ctx, all evals) ⇒ no extra masks, ++ // no extra VRAM, kernel gets nullptr ⇒ #617 analytical fast path stays byte-identical. ++ bool fragmented = mctx_kv->get_is_fragmented(); ++ ++ // opencoti F5 dca (#623): MROPE/IMROPE archs (qwen35/qwen35moe) need 4 position ids per token so the ++ // per-regime Q ropes through ggml_rope_multi with the model's section split; NEOX archs use 1. The ++ // mask + pos_q_real shapes are position-COUNT based (n_tokens), unaffected — only pos_q widens. ++ const int64_t npp = dca_pos_per_token(rope_type); ++ ++ // opencoti F5 dca perf (T87.pD-gen #617): pos_k (regime-invariant key positions, mod c) is NO ++ // LONGER allocated. The cache now stores PRE-roped K (roped once at insert in build_attn_dca), so ++ // build_attn_dca_core reads it directly and nothing consumes pos_k. Leaving it nullptr avoids an ++ // orphan graph input (registered-but-unread => null buffer => set_input writes null->data — the ++ // same DCA_INTRA_ONLY segfault class guarded below). set_input_dca null-skips the pos_k fill. ++ ++ static const char * rname[DCA_REGIME_COUNT] = { "dca_mask_intra", "dca_mask_succ", "dca_mask_inter" }; ++ for (int r = 0; r < DCA_REGIME_COUNT; ++r) { ++ // opencoti F5 dca perf (T87.pD-opt Stage 1): single-chunk serving (n_ctx <= c) only ever uses ++ // the INTRA regime — every query is in chunk 0 so SUCC/INTER masks are all -inf. Leave their ++ // pos_q/mask tensors null so they are NOT registered as graph inputs: a registered-but-unread ++ // input gets a null buffer from the allocator and set_input_dca would write to null->data ++ // (the DCA_INTRA_ONLY segfault). build_attn_dca_core runs the INTRA-only fast path to match. ++ if (inp->single_chunk && r != DCA_REGIME_INTRA) { ++ continue; ++ } ++ inp->pos_q[r] = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * npp); ++ ggml_set_input(inp->pos_q[r]); ++ ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): SUCC/INTER under analytical_bands need NO dense ++ // mask — the CUDA _lse op derives their contiguous KV band on-GPU from pos_q + chunk. Leave ++ // mask[r]/mask_cnv[r] null (the FA call passes mask=nullptr for them), killing the host ++ // O(n_kv*n_tokens) fill + multi-GB H2D for 2 of 3 regimes. pos_q[r] above is still required: it ++ // ropes Q for the regime AND seeds the on-GPU band. INTRA always keeps its native-causal mask. ++ // opencoti F5 dca (#617B): on the FRAGMENTED multi-chunk path we DO allocate the SUCC/INTER ++ // dense masks (index-based bands are invalid when index != pos) so set_input_dca can fill them ++ // position-based and build_attn_dca_core feeds them to the fused kernel. Contiguous cache ++ // (the always-case) keeps skipping them ⇒ identical to #618/#617 fast path. ++ if (inp->analytical_bands && r != DCA_REGIME_INTRA && !fragmented) { ++ continue; ++ } ++ ++ // additive mask [n_kv, n_tokens, 1, 1] — single-stream. Mirrors build_attn_inp_kq_mask's ++ // unpadded shape; cast to F16 for the flash-attn op when FA is enabled. ++ inp->mask[r] = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1, 1); ++ ggml_set_input(inp->mask[r]); ++ ggml_set_name(inp->mask[r], rname[r]); ++ ++ inp->mask_cnv[r] = cparams.flash_attn ++ ? ggml_cast(ctx0, inp->mask[r], GGML_TYPE_F16) ++ : inp->mask[r]; ++ } ++ ++ // opencoti F5 dca O6-fix (>=3 chunks): the analytical band needs the REAL absolute query position ++ // (cq = real_pos/chunk). The per-regime pos_q the kernel received was RoPE-remapped (succ->oq+c, ++ // inter->c const) so cq==1 for every query at chunk>=2 — SUCC collapsed to the chunk-0 band, INTER ++ // emptied. Allocate one regime-independent real-position channel; only the multi-chunk analytical ++ // path reads it (the SUCC + INTER _lse ops). single_chunk / dense-mask paths leave it nullptr (no ++ // orphan input — set_input_dca null-guards the fill). ++ if (inp->analytical_bands) { ++ inp->pos_q_real = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); ++ ggml_set_input(inp->pos_q_real); ++ ggml_set_name(inp->pos_q_real, "dca_pos_q_real"); ++ } ++ ++ return (llm_graph_input_dca *) res->add_input(std::move(inp)); ++} ++ ++// Shared DCA attention core. The caller has stored PRE-roped K (#617: roped once at insert) + un-roped ++// V to the cache (mctx_cur). q_cur is PRE-rope (post-norm) [n_embd_head, n_head, n_tokens]. Reads the ++// cached K directly, ropes the small Q per regime, and runs the fused single-pass analytical-band FA ++// (multi-chunk) or plain FA (single_chunk) before the wo projection. The legacy 3x flash_attn_ext_lse ++// + host LSE-combine path was removed (#618). See docs/features/gemma4_dca.md Section C. ++ ++// opencoti F5 dca (T87.pD-qwen35 #623): DCA remaps the query position per regime, but qwen35/qwen35moe ++// carry rope_type = GGML_ROPE_TYPE_IMROPE (=40, MROPE bit set), whose rope op asserts 4 ids/token ++// (a->ne[2]*4 == b->ne[0]) and assigns INTERLEAVED per-section frequencies across head dims. The early ++// NEOX-collapse (rope MROPE as plain NEOX with a scalar position) shape-fixed the assert but produced ++// GARBAGE: IMRoPE's interleaved freq->dim map is NOT equal to NEOX even when every section sits at the ++// same text position (the single-chunk ±DCA gate diverged at char 21 into a token loop). The faithful ++// path below ropes the DCA-remapped position through ggml_rope_multi with the model's own section split ++// — for TEXT all spatial sections sit at the temporal position (sections 0-2 = remapped pos, section 3 ++// = 0), exactly the layout llm_graph_input_pos::set_input builds for native MROPE text. NEOX/NORMAL ++// archs (gemma4/qwen2/qwen3) keep ggml_rope_ext (1-wide pos) — (rope_type & MROPE)==0. The ++// dca_is_mrope / dca_pos_per_token predicates this relies on are defined near the top of the file. ++ ++// Rope the DCA per-regime Q (or insert-K) faithfully for the arch. `pos` must be the width ++// build_attn_inp_dca allocated: 4*n_tokens for MROPE (section split below), n_tokens for NEOX. ++ggml_tensor * llm_graph_context::build_dca_rope(ggml_tensor * x, ggml_tensor * pos, const dca_rope & rp) const { ++ if (dca_is_mrope(rope_type)) { ++ int sections[GGML_MROPE_SECTIONS] = { rp.sections[0], rp.sections[1], rp.sections[2], rp.sections[3] }; ++ return ggml_rope_multi(ctx0, x, pos, rp.freq_factors, ++ rp.n_rot, sections, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++ } ++ return ggml_rope_ext(ctx0, x, pos, rp.freq_factors, ++ rp.n_rot, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++} ++ ++ggml_tensor * llm_graph_context::build_attn_dca_core( ++ const llama_kv_cache_context * mctx_cur, ++ llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ++ const dca_rope & rp, ++ float kq_scale, int il) const { ++ const float max_bias = hparams.f_max_alibi_bias; ++ const float softcap = hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f; ++ ++ // (1) read the cached K. opencoti F5 dca perf (T87.pD-gen #617): K is now PRE-ROPED at insert ++ // (rope-at-(pos mod c), applied once to the small new-token K in build_attn_dca below) and ++ // kept correct under context-shift by the DCA-aware modular K-shift. So read it DIRECTLY — no ++ // per-step full-cache rope. That re-rope of the entire cache every graph eval was the ++ // O(n_kv)/token decode bottleneck (gen halving per context-doubling). get_k returns ++ // [n_embd_head, n_head_kv, n_kv, 1]. DCA requires an f16 K cache (no q8_0/turbo on this path). ++ ggml_tensor * k_dca = mctx_cur->get_k(ctx0, il); ++ GGML_ASSERT(k_dca->type == GGML_TYPE_F16 && "dca: requires an f16 K cache (no q8_0/turbo KV)"); ++ ++ // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. ++ ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ const bool v_trans = v->nb[1] > v->nb[2]; ++ ggml_tensor * vp = ggml_permute(ctx0, v, 0, 2, 1, 3); ++ if (v_trans) { ++ vp = ggml_transpose(ctx0, vp); ++ } ++ if (vp->type == GGML_TYPE_F32) { ++ vp = ggml_cast(ctx0, vp, GGML_TYPE_F16); ++ } ++ ggml_tensor * kp = ggml_permute(ctx0, k_dca, 0, 2, 1, 3); ++ ++ // (3) Q-rope + attention. Two sub-paths (the legacy non-analytical multi-chunk fallback was removed ++ // in #618 — its precondition is enforced at context init): ++ // single_chunk → plain optimized FA (INTRA covers all keys, ordinary positions ⇒ exact full ++ // attention; GGML_PREC_F32 mirrors the native global-layer call). ++ // analytical bands → ONE fused single-pass FA (T87.pD Branch B): the 3 pre-roped Q variants feed ++ // a kernel that walks the partitioned causal range in INTER→SUCC→INTRA key- ++ // phases under one continuous online-softmax — NO per-regime ops, NO host LSE ++ // combine. analytical_bands == !single_chunk; requires cq uniform per ubatch ++ // (chunk_size % n_ubatch == 0), enforced at context init. ++ ggml_tensor * cur; ++ if (inp_dca->single_chunk) { ++ ggml_tensor * qr = build_dca_rope(q_cur, inp_dca->get_pos_q(DCA_REGIME_INTRA), rp); ++ ggml_tensor * qp = ggml_permute(ctx0, qr, 0, 2, 1, 3); ++ ggml_tensor * O = ggml_flash_attn_ext(ctx0, qp, kp, vp, ++ inp_dca->get_mask(DCA_REGIME_INTRA), kq_scale, max_bias, softcap); ++ ggml_flash_attn_ext_set_prec(O, GGML_PREC_F32); ++ cur = O; // [DV, H, NQ, 1] normalized ++ } else if (inp_dca->analytical_bands) { ++ // pre-rope the 3 Q variants (K/V already roped once above); permute to FA layout. ++ ggml_tensor * qp[DCA_REGIME_COUNT]; ++ for (int r = 0; r < DCA_REGIME_COUNT; ++r) { ++ ggml_tensor * qr = build_dca_rope(q_cur, inp_dca->get_pos_q((dca_regime) r), rp); ++ qp[r] = ggml_permute(ctx0, qr, 0, 2, 1, 3); ++ } ++ // one fused pass: INTER→SUCC→INTRA key-phases; INTRA mask bounds the diagonal; pos_q_real ++ // (REAL abs query pos) is the band channel the kernel reads to derive cq = real_pos/chunk. ++ // opencoti F5 dca (#617B): get_mask(SUCC)/get_mask(INTER) are nullptr on the contiguous fast ++ // path (analytical bands) and the dense position-based F16 masks when the cache is fragmented; ++ // when non-null the kernel switches each phase to full-range + dense-mask selection. ++ ggml_tensor * P = ggml_flash_attn_ext_dca_fused(ctx0, ++ qp[DCA_REGIME_INTRA], qp[DCA_REGIME_SUCC], qp[DCA_REGIME_INTER], ++ kp, vp, inp_dca->get_mask(DCA_REGIME_INTRA), ++ inp_dca->get_mask(DCA_REGIME_SUCC), inp_dca->get_mask(DCA_REGIME_INTER), ++ inp_dca->get_pos_q_real(), ++ kq_scale, max_bias, softcap, (int) inp_dca->chunk_size); ++ // dst is [DV+1, H, NQ, 1] (dense O block then UNUSED lse row); slice the dense O block. ++ const int64_t DV = P->ne[0] - 1; ++ const int64_t H = P->ne[1]; ++ const int64_t NQ = P->ne[2]; ++ const size_t ts = P->nb[0]; ++ cur = ggml_cont(ctx0, ggml_view_4d(ctx0, P, DV, H, NQ, 1, ++ DV*ts, DV*H*ts, DV*H*NQ*ts, 0)); ++ } else { ++ // opencoti F5 dca (#618): the legacy non-analytical multi-chunk path (3 per-regime ++ // ggml_flash_attn_ext_lse ops at regime 0 + a host f32 online-softmax combine) was REMOVED. It ++ // was a CUDA-only fallback for chunk_size % n_ubatch != 0, never ran in production, and degraded ++ // output. That precondition is now enforced at context init (llama_context::llama_context ++ // throws), so analytical_bands == !single_chunk and this branch is unreachable — abort ++ // defensively if the invariant is ever violated. ++ GGML_ABORT("dca: non-analytical multi-chunk reached — chunk_size must be a multiple of n_ubatch " ++ "(enforced at context init; legacy fallback removed, #618)"); ++ } ++ cb(cur, "kqv_out", il); ++ ++ // (5) merge heads -> [n_embd, n_tokens] and apply the output projection. ++ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]); ++ if (wo) { ++ cur = build_lora_mm(wo, cur); ++ } ++ if (wo_b) { ++ cur = ggml_add(ctx0, cur, wo_b); ++ } ++ return cur; ++} ++ ++// iswa overload — Gemma-4 GLOBAL (non-SWA) layers. DCA never runs on SWA layers (local window <= c ++// already, no chunking needed). Stores PRE-rope K + un-roped V to the BASE cache; has_kv==false ++// shared-KV global layers pass k_cur==v_cur==nullptr and just read the earlier layer's cache. ++ggml_tensor * llm_graph_context::build_attn_dca( ++ llm_graph_input_attn_kv_iswa * inp, ++ llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ggml_tensor * k_cur, ggml_tensor * v_cur, ++ const dca_rope & rp, float kq_scale, int il) const { ++ GGML_ASSERT(!hparams.is_swa(il) && "dca: only global (non-SWA) layers route through build_attn_dca"); ++ ++ const auto * mctx_cur = inp->mctx->get_base(); ++ ++ ggml_build_forward_expand(gf, q_cur); ++ if (k_cur) { ggml_build_forward_expand(gf, k_cur); } ++ if (v_cur) { ggml_build_forward_expand(gf, v_cur); } ++ ++ // opencoti F5 dca perf (T87.pD-gen #617): PRE-ROPE the small new-token K once, HERE, with the ++ // regime-invariant key positions pos mod c, then store the roped K. pos_q[INTRA] is filled with ++ // exactly ubatch->pos[i] % c (set_input_dca), which equals get_pos_k() for a freshly inserted cell ++ // — so the cache holds bit-identically what build_attn_dca_core's (now dropped) full-cache K-rope ++ // would have produced. Decode then reads pre-roped K with no O(n_kv) per-step rope. ++ if (k_cur) { ++ ggml_tensor * k_ins = build_dca_rope(k_cur, inp_dca->get_pos_q(DCA_REGIME_INTRA), rp); ++ ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_ins, inp->get_k_idxs(), il)); ++ } ++ if (v_cur) { ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, inp->get_v_idxs(), il)); } ++ ++ return build_attn_dca_core(mctx_cur, inp_dca, wo, wo_b, q_cur, rp, kq_scale, il); ++} ++ ++// plain-kv overload — Qwen (all layers are full attention). ++ggml_tensor * llm_graph_context::build_attn_dca( ++ llm_graph_input_attn_kv * inp, ++ llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ggml_tensor * k_cur, ggml_tensor * v_cur, ++ const dca_rope & rp, float kq_scale, int il) const { ++ const auto * mctx_cur = inp->mctx; ++ ++ ggml_build_forward_expand(gf, q_cur); ++ if (k_cur) { ggml_build_forward_expand(gf, k_cur); } ++ if (v_cur) { ggml_build_forward_expand(gf, v_cur); } ++ ++ // opencoti F5 dca perf (T87.pD-gen #617): PRE-ROPE the new-token K once at insert (pos mod c via ++ // pos_q[INTRA]); see the iswa overload above for the full rationale. Decode reads pre-roped K. ++ if (k_cur) { ++ ggml_tensor * k_ins = build_dca_rope(k_cur, inp_dca->get_pos_q(DCA_REGIME_INTRA), rp); ++ ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_ins, inp->get_k_idxs(), il)); ++ } ++ if (v_cur) { ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, inp->get_v_idxs(), il)); } ++ ++ return build_attn_dca_core(mctx_cur, inp_dca, wo, wo_b, q_cur, rp, kq_scale, il); ++} +diff --git a/llama.cpp/src/dca.h b/llama.cpp/src/dca.h +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/src/dca.h +@@ -0,0 +1,122 @@ ++#pragma once ++ ++// opencoti F5 dca (#593, T87.pD) — Dual Chunk Attention: training-free long-context. ++// Design + rationale: docs/features/gemma4_dca.md (Section C). This header declares the ++// graph input that carries the 3-regime DCA position remap (built once per ubatch, host-filled ++// from the batch + KV-cell positions) plus the small helpers shared by the two build_attn_dca ++// overloads (in dca.cpp). The LSE-emitting flash-attn op it feeds is ggml_flash_attn_ext_lse ++// (Section D, ggml.h). The build_attn_dca methods themselves are declared on llm_graph_context ++// in llama-graph.h (so they can reach build_lora_mm / ctx0 / gf like the other build_attn ops). ++ ++#include "llama-graph.h" // llm_graph_input_i, llm_graph_params ++#include "llama-hparams.h" ++#include "llama-cparams.h" ++ ++#include ++ ++class llama_kv_cache_context; ++ ++// DCA position regimes. The three keep-masks partition the causal lower-triangle exactly, so the ++// LSE-merge of the three per-regime softmaxes equals full attention under DCA-remapped positions. ++// intra: query/key in the same chunk — pos_q = i mod c (distance < c) ++// succ : key in the immediately-prior chunk — pos_q = (i mod c) + c (distance in (0, 2c)) ++// inter: key in any older chunk — pos_q = c (constant) (distance = c - ok) ++// All three share pos_k[j] = cellpos(j) mod c, so the (large) cached K is roped ONCE; only the ++// (small) Q is roped per regime. See docs/features/gemma4_dca.md "rope-K-ONCE, rope-Q-THRICE". ++enum dca_regime { ++ DCA_REGIME_INTRA = 0, ++ DCA_REGIME_SUCC = 1, ++ DCA_REGIME_INTER = 2, ++ DCA_REGIME_COUNT = 3, ++}; ++ ++// Per-layer rope parameters threaded into build_attn_dca. Section B fills these from the model ++// (freq_base = model.get_rope_freq_base(cparams, il), freq_scale likewise, n_rot = hparams.n_rot(il), ++// freq_factors = model.layers[il].rope_freqs). The global YaRN knobs (ext/attn/beta_*) + rope_type + ++// n_ctx_orig come from the llm_graph_context directly, matching the per-arch model rope call. ++struct dca_rope { ++ float freq_base = 0.0f; ++ float freq_scale = 1.0f; ++ int n_rot = 0; ++ ggml_tensor * freq_factors = nullptr; ++ // opencoti F5 dca (#623) — MROPE/IMROPE section split (hparams.rope_sections), e.g. [11,11,10,0] ++ // for qwen35/qwen35moe. Used only when rope_type carries the MROPE bit (build_dca_rope picks ++ // ggml_rope_multi); NEOX/NORMAL archs leave it {0,0,0,0} and rope through ggml_rope_ext. ++ int sections[4] = { 0, 0, 0, 0 }; ++}; ++ ++// Resolve the effective chunk length c: cparams.dca_chunk_size when non-zero, else (auto) the ++// model's original/pretrain context window (n_ctx_orig_yarn, falling back to n_ctx_train). c must ++// be >= 1; callers gate on cparams.dca_enabled before constructing the input. ++uint32_t dca_resolve_chunk_size(const llama_cparams & cparams, const llama_hparams & hparams); ++ ++// Per-ubatch DCA position input. Lives on the BASE kv-cache context (the non-SWA one for Gemma-4 ++// global layers via build_attn_inp_kv_iswa()->get_base(), or the plain kv context for Qwen). ++class llm_graph_input_dca : public llm_graph_input_i { ++public: ++ llm_graph_input_dca( ++ const llama_hparams & hparams, ++ const llama_cparams & cparams, ++ const llama_kv_cache_context * mctx, ++ uint32_t chunk_size) : ++ hparams(hparams), ++ cparams(cparams), ++ mctx(mctx), ++ chunk_size(chunk_size), ++ single_chunk(cparams.n_ctx <= chunk_size), ++ // opencoti F5 dca (#618): the multi-chunk path is ALWAYS the fused analytical-band kernel, so ++ // analytical_bands is simply !single_chunk. Its precondition — chunk_size % n_ubatch == 0, so ++ // every query in a ubatch shares cq = pos/c and each regime's KV band is one contiguous range ++ // (no per-element causal, no chunk-straddling tile) — is enforced at context init ++ // (llama_context::llama_context throws otherwise), so !single_chunk ⇒ analytical holds. The old ++ // dense-mask "legacy" 3x flash_attn_ext_lse + HOST f32 online-softmax combine fallback was ++ // REMOVED (#618): it never ran in prod, was CUDA-only (not a portability path), and degraded ++ // output. Future non-CUDA backends port the fused kernel, not the legacy host combine. ++ analytical_bands(!(cparams.n_ctx <= chunk_size)) { ++ } ++ ~llm_graph_input_dca() = default; ++ ++ void set_input(const llama_ubatch * ubatch) override; ++ ++ bool can_reuse(const llm_graph_params & params) override; ++ ++ // Regime-invariant key positions: pos_k[j] = cellpos(j) mod chunk_size. I32 [n_kv]. ++ ggml_tensor * get_pos_k() const { return pos_k; } ++ // Per-regime query positions. I32 [n_tokens]. ++ ggml_tensor * get_pos_q(dca_regime r) const { return pos_q[r]; } ++ // O6-fix (>=3 chunks): REAL absolute query positions (= ubatch->pos[i]), regime-independent. ++ // Passed as the analytical-band side-channel (NOT for RoPE) so the kernel derives the TRUE ++ // cq = real_pos/chunk. get_pos_q(r) is RoPE-remapped (succ -> oq+c, inter -> c const) and yields ++ // cq==1 for every query at chunk>=2 -> SUCC band collapses to chunk 0, INTER band empties. I32 [n_tokens]. ++ ggml_tensor * get_pos_q_real() const { return pos_q_real; } ++ // Per-regime additive mask, already cast to the flash-attn dtype. [n_kv, pad(n_tps), 1, n_stream]. ++ ggml_tensor * get_mask(dca_regime r) const { return mask_cnv[r]; } ++ ++ ggml_tensor * pos_k = nullptr; // I32 [n_kv] ++ ggml_tensor * pos_q [DCA_REGIME_COUNT] = { nullptr, nullptr, nullptr }; // I32 [n_tokens] ++ ggml_tensor * pos_q_real = nullptr; // I32 [n_tokens] real abs query pos (band channel, O6-fix) ++ ggml_tensor * mask [DCA_REGIME_COUNT] = { nullptr, nullptr, nullptr }; // F32 [n_kv, pad, 1, n_stream] ++ ggml_tensor * mask_cnv [DCA_REGIME_COUNT] = { nullptr, nullptr, nullptr }; // F16 (flash-attn) or = mask ++ ++ const llama_kv_cache_context * mctx = nullptr; ++ ++ // copies (graph-reuse safety, mirrors llm_graph_input_attn_kv) — see llama-graph.h note. ++ const llama_hparams hparams; ++ const llama_cparams cparams; ++ const uint32_t chunk_size; ++ ++ // opencoti F5 dca perf (T87.pD-opt Stage 1): n_ctx <= chunk_size => every query is in chunk 0, ++ // so SUCC/INTER regimes are always empty and the 3-regime result == INTRA-only (plain attention). ++ // When set, build_attn_inp_dca allocates only the INTRA pos_q/mask (SUCC/INTER stay nullptr — NOT ++ // orphan graph inputs), set_input_dca fills only INTRA, and build_attn_dca_core runs one regime ++ // and skips the LSE combine. Eliminates 2 of 3 flash-attn + lse-recompute passes per global layer. ++ const bool single_chunk; ++ ++ // opencoti F5 dca perf (T87.pD-opt O6 Tier B): when true, the SUCC/INTER regimes use ANALYTICAL ++ // per-tile KV band-bounds (computed on-GPU from pos_q + chunk) instead of a dense [n_kv x n_tokens] ++ // mask — killing 2 of 3 host O(n_kv*n_tokens) mask builds + their multi-GB H2D. build_attn_inp_dca ++ // then skips the SUCC/INTER mask allocation (pos_q stays — it ropes Q AND seeds the bounds), and ++ // build_attn_dca_core passes mask=nullptr + pos_q + chunk + regime to ggml_flash_attn_ext_lse for ++ // those two. INTRA always keeps its native-causal mask (the irreducible native-equivalent cost). ++ const bool analytical_bands; ++}; +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -10,6 +10,7 @@ + #include "llama-mmap.h" + #include "llama-model.h" + #include "llama-ext.h" ++#include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" + + #include +@@ -18,6 +19,12 @@ + #include + #include + ++#include ++#include ++#include ++#include ++#include ++ + // + // llama_context + // +@@ -74,6 +81,10 @@ llama_context::llama_context( + cparams.neo_pipeline_mode = params.neo_pipeline_mode; + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md + cparams.fused_moe_up_gate = params.fused_moe_up_gate; ++ // opencoti F5 dca — see docs/features/gemma4_dca.md ++ cparams.dca_enabled = params.dca_enabled; ++ cparams.dca_chunk_size = params.dca_chunk_size; ++ cparams.dca_yarn_factor = params.dca_yarn_factor; + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -199,6 +210,21 @@ llama_context::llama_context( + + cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); + ++ // opencoti F5 dca (#618): the legacy non-analytical multi-chunk DCA fallback was removed, so DCA now ++ // REQUIRES chunk_size % n_ubatch == 0 — the fused analytical-band kernel needs every query in a ++ // ubatch to share cq = pos/chunk (one contiguous KV band per regime). Enforce it here: n_ubatch is ++ // final, and the effective chunk size is resolvable (dca_enabled/dca_chunk_size set above, ++ // n_ctx_orig_yarn at ~:81). Fail loud rather than silently degrade output (the removed fallback ++ // produced malformed generations on a chunk%ubatch != 0 config). ++ if (cparams.dca_enabled) { ++ const uint32_t dca_c = dca_resolve_chunk_size(cparams, hparams); ++ if (cparams.n_ubatch == 0 || dca_c % cparams.n_ubatch != 0) { ++ throw std::runtime_error( ++ "DCA: dca chunk size (" + std::to_string(dca_c) + ") must be a multiple of n_ubatch (" + ++ std::to_string(cparams.n_ubatch) + "); set -ub/--ubatch-size to a divisor of the chunk size"); ++ } ++ } ++ + cparams.op_offload = params.op_offload; + cparams.kv_unified = params.kv_unified; + +@@ -3385,6 +3411,9 @@ llama_context_params llama_context_default_params() { + /*.kv_residency_mode =*/ 0, // opencoti F5 M7 Rolling KV — 0 = auto + /*.neo_pipeline_mode =*/ 0, // opencoti F5 M3 neo — off by default + /*.fused_moe_up_gate =*/ false, // opencoti F5 W3 #292 — off by default ++ /*.dca_enabled =*/ false, // opencoti F5 dca — off by default ++ /*.dca_chunk_size =*/ 0, // opencoti F5 dca — 0 = auto (<- n_ctx_orig_yarn) ++ /*.dca_yarn_factor =*/ 1.0f, // opencoti F5 dca — 1.0 = no YaRN stretch + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -48,6 +48,14 @@ struct llama_cparams { + // true = build_moe_ffn collapses the up/gate expert matmuls + GLU into one + // GGML_OP_MOE_FUSED_UP_GATE op at decode (n_tokens==1). false = unfused path. + bool fused_moe_up_gate; ++ // opencoti F5 dca — Dual Chunk Attention training-free long-context (docs/features/gemma4_dca.md). ++ // dca_enabled: route full-attention layers (Gemma4 global / Qwen all) through build_attn_dca ++ // (cache K un-roped, 3-regime rope-on-read + streaming_combine_n). false = off (default). ++ // dca_chunk_size: chunk length c; 0 = auto (<- n_ctx_orig_yarn pretrain window). ++ // dca_yarn_factor: YaRN mscale stretch composed with DCA; 1.0 = none. ++ bool dca_enabled; ++ uint32_t dca_chunk_size; ++ float dca_yarn_factor; + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -107,7 +107,13 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) { + } + + void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { +- if (ubatch->pos && pos) { ++ // opencoti F5 dca #623: in a DCA-on graph the full-attention layers rope Q from inp_dca's own ++ // per-regime positions (build_attn_dca), so the standard inp_pos is consumed by no node on archs ++ // where EVERY attention layer is DCA-routed (qwen2/qwen3/qwen3moe; qwen35/qwen35moe whose SSM ++ // layers are positionless). galloc then leaves `pos` with a NULL buffer and the unguarded ++ // ggml_backend_tensor_set below asserts buf!=NULL (ggml-backend.cpp:297). Guard on a live buffer. ++ // gemma4-iswa is unaffected: its SWA layers still rope inp_pos, so pos is always consumed. ++ if (ubatch->pos && pos && pos->buffer) { + const int64_t n_tokens = ubatch->n_tokens; + + if (ubatch->token && n_pos_per_embd == 4) { +@@ -454,16 +460,22 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { + } + + void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) { +- mctx->set_input_k_idxs(self_k_idxs, ubatch); +- mctx->set_input_v_idxs(self_v_idxs, ubatch); ++ // opencoti F5 dca #623: when DCA diverts the full-attention layers to build_attn_dca (which builds ++ // its own dca_mask_* and pre-ropes K at insert), the standard K/V write-index and K-shift rot ++ // tensors are consumed by no compute node, so galloc leaves them with a NULL buffer. Writing them ++ // then asserts in set_input_k_idxs's is_host(dst->buffer) (bug-404). Guard every idx/rot setter on ++ // a live buffer — exactly the iswa (gemma4) path's guard. (set_input_kq_mask self-guards on a null ++ // buffer.) Non-DCA graphs: the tensors are always consumed, so the guards are no-ops. ++ if (self_k_idxs && self_k_idxs->buffer) { mctx->set_input_k_idxs(self_k_idxs, ubatch); } ++ if (self_v_idxs && self_v_idxs->buffer) { mctx->set_input_v_idxs(self_v_idxs, ubatch); } + + mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + +- if (self_k_rot) { ++ if (self_k_rot && self_k_rot->buffer) { + mctx->set_input_k_rot(self_k_rot); + } + +- if (self_v_rot) { ++ if (self_v_rot && self_v_rot->buffer) { + mctx->set_input_v_rot(self_v_rot); + } + } +@@ -593,16 +605,21 @@ void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { + } + + void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { +- mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch); +- mctx->get_attn()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch); ++ // opencoti F5 dca #623: same orphan-buffer guard as the iswa path. DCA diverts the hybrid's ++ // full-attention layers to build_attn_dca, leaving the standard idx/rot tensors unconsumed ++ // (NULL buffer); the unguarded setters then assert is_host(NULL). Guard each on a live buffer. ++ // (kq_mask self-guards.) qwen35moe routes EVERY full-attention layer through DCA, so unlike ++ // gemma4-iswa (SWA layers keep the masks live) nothing else consumes these — the guard is load-bearing. ++ if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) { mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch); } ++ if (inp_attn->self_v_idxs && inp_attn->self_v_idxs->buffer) { mctx->get_attn()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch); } + + mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn); + +- if (inp_attn->self_k_rot) { ++ if (inp_attn->self_k_rot && inp_attn->self_k_rot->buffer) { + mctx->get_attn()->set_input_k_rot(inp_attn->self_k_rot); + } + +- if (inp_attn->self_v_rot) { ++ if (inp_attn->self_v_rot && inp_attn->self_v_rot->buffer) { + mctx->get_attn()->set_input_v_rot(inp_attn->self_v_rot); + } + +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -27,6 +27,9 @@ class llama_memory_recurrent_context; + class llama_memory_hybrid_context; + class llama_memory_hybrid_iswa_context; + ++class llm_graph_input_dca; // opencoti F5 dca #593 (defined in dca.h) ++struct dca_rope; // opencoti F5 dca #593 (defined in dca.h) ++ + // certain models (typically multi-modal) can produce different types of graphs + enum llm_graph_type { + LLM_GRAPH_TYPE_DEFAULT, +@@ -998,6 +1001,34 @@ struct llm_graph_context { + + llm_graph_input_attn_kv * build_attn_inp_kv() const; + ++ // opencoti F5 dca (#593) — DCA 3-regime position-remap input (src/dca.cpp). mctx_kv = the ++ // BASE kv-cache context (iswa get_base() for Gemma-4 globals / the plain kv context for Qwen). ++ llm_graph_input_dca * build_attn_inp_dca(const llama_kv_cache_context * mctx_kv) const; ++ ++ // opencoti F5 dca (#593) — DCA attention (3-regime chunked-position) for one layer (src/dca.cpp). ++ // Two overloads mirror the build_attn input types: iswa (Gemma-4 globals) + plain-kv (Qwen). ++ // q_cur and k_cur are PRE-rope (Section B passes them un-roped); rope happens per regime inside. ++ ggml_tensor * build_attn_dca( ++ llm_graph_input_attn_kv_iswa * inp, llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ggml_tensor * k_cur, ggml_tensor * v_cur, ++ const dca_rope & rp, float kq_scale, int il) const; ++ ggml_tensor * build_attn_dca( ++ llm_graph_input_attn_kv * inp, llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ggml_tensor * k_cur, ggml_tensor * v_cur, ++ const dca_rope & rp, float kq_scale, int il) const; ++ // shared core: K/V already stored by the caller; mctx_cur is the BASE kv-cache context. ++ ggml_tensor * build_attn_dca_core( ++ const llama_kv_cache_context * mctx_cur, llm_graph_input_dca * inp_dca, ++ ggml_tensor * wo, ggml_tensor * wo_b, ggml_tensor * q_cur, ++ const dca_rope & rp, float kq_scale, int il) const; ++ // opencoti F5 dca (#623) — rope the DCA per-regime Q (or the insert-K) with the op that matches ++ // the arch: ggml_rope_multi (MROPE/IMROPE — qwen35/qwen35moe; pos is 4-wide, model section split) ++ // or ggml_rope_ext (NEOX/NORMAL — gemma4/qwen2/qwen3; pos 1-wide). `pos` MUST be the width ++ // build_attn_inp_dca allocated for this arch (n_tokens or 4*n_tokens). See src/dca.cpp. ++ ggml_tensor * build_dca_rope(ggml_tensor * x, ggml_tensor * pos, const dca_rope & rp) const; ++ + ggml_tensor * build_attn( + llm_graph_input_attn_kv * inp, + ggml_tensor * wo, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -4,6 +4,7 @@ + #include "llama-io.h" + #include "llama-model.h" + #include "llama-context.h" ++#include "dca.h" // opencoti F5 dca (#593) — dca_resolve_chunk_size for the DCA-aware K-shift + + #include + #include +@@ -2142,6 +2143,20 @@ bool llama_kv_cache::get_has_shift() const { + return result; + } + ++// opencoti F5 dca (#617B): true if ANY stream's cells have been position-relabeled in place (a ++// classic context-shift / self-extend), so cell index != absolute position. The DCA host-fill reads ++// this at graph-build time to gate the position-based dense-mask fallback (the analytical band ++// selection is index-based and silently wrong on a non-contiguous cache). See llama_kv_cells. ++bool llama_kv_cache::get_is_fragmented() const { ++ bool result = false; ++ ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ result |= v_cells[s].get_is_fragmented(); ++ } ++ ++ return result; ++} ++ + ggml_type llama_kv_cache::type_k() const { + // opencoti F4 M3 Phase 4: all per-stream tensors of a layer carry the + // same type — read it from stream 0. Works in both unified and +@@ -3547,6 +3562,36 @@ void llama_kv_cache::set_input_k_shift(ggml_tensor * dst) const { + } + } + ++// opencoti F5 dca (#593, T87.pD-gen #617) — modular K-shift delta for DCA global layers. They cache ++// PRE-roped K at rope-at-(pos mod c) (roped once on the new-token K at insert, in build_attn_dca). A ++// context-shift moves a cell from pos_old -> pos_new, so its K rotation must move (pos_old mod c) -> ++// (pos_new mod c). RoPE composes additively, so build_rope_shift applies rope-by-delta to the cached ++// K; for DCA the delta is the MODULAR difference (pos_new mod c) - (pos_old mod c), NOT the native ++// absolute pos_new - pos_old. Both are host-side here: get_shift(i) == pos_new - pos_old (the native ++// channel), so pos_old = pos_new - get_shift(i). Empty cells contribute 0 (no rotation). ++void llama_kv_cache::set_input_k_shift_dca(ggml_tensor * dst, uint32_t chunk_size) const { ++ GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); ++ const int64_t c = (int64_t) chunk_size; ++ GGML_ASSERT(c >= 1 && "dca: K-shift needs a positive chunk size"); ++ ++ int32_t * data = (int32_t *) dst->data; ++ ++ for (uint32_t s = 0; s < n_stream; ++s) { ++ const auto & cells = v_cells[s]; ++ ++ for (uint32_t i = 0; i < cells.size(); ++i) { ++ if (cells.is_empty(i)) { ++ data[s*cells.size() + i] = 0; ++ continue; ++ } ++ const llama_pos pnew = cells.pos_get(i); ++ const llama_pos pold = pnew - cells.get_shift(i); // position the cached K is currently roped at ++ // pnew, pold are >= 0, so the C++ % is non-negative; delta range is (-(c-1), c-1). ++ data[s*cells.size() + i] = (int32_t) (pnew % c) - (int32_t) (pold % c); ++ } ++ } ++} ++ + struct args_set_input_kq_mask { + const llama_hparams & hparams; + const llama_ubatch * ubatch; +@@ -3741,6 +3786,15 @@ static void set_input_kq_mask_impl(const args_set_input_kq_mask & args, float * + void llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const { + const uint32_t n_tokens = ubatch->n_tokens; + ++ // opencoti F5 dca #593: when DCA is active the global/full-attention layers are diverted to ++ // build_attn_dca (which builds its own dca_mask_* tensors), so the standard kq_mask for those ++ // layers (gemma4 self_kq_mask; qwen self_kq_mask) is consumed by no compute node and the graph ++ // allocator leaves it with a NULL buffer. It is a pure orphan — nothing reads it — so there is ++ // nothing to fill. Skip it. (In non-DCA graphs this never triggers: the mask is always used.) ++ if (dst->buffer == NULL) { ++ return; ++ } ++ + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + float * data = (float *) dst->data; + +@@ -3802,6 +3856,185 @@ void llama_kv_cache::set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch + } + } + ++// opencoti F5 dca (#593) — host-fill the DCA 3-regime position remap. Single-stream only (a 1M ++// context is single-sequence; multi-stream is a TODO, mirroring set_input_pos_bucket). Fills: ++// pos_k[j] = cellpos(j) mod c (regime-invariant key positions; empty cell -> 0) ++// pos_q_*[i] per regime: intra=oq, succ=oq+c, inter=c (oq = pos(i) mod c) ++// mask_*[i,j] = 0 where (causal p0<=p1, same-seq, nonempty) AND chunk(j) in the regime band, ++// else -inf. The three masks partition the causal lower-triangle exactly, so the ++// LSE-merge of the three softmaxes equals full attention under DCA positions. ++void llama_kv_cache::set_input_dca( ++ ggml_tensor * pos_k, ++ ggml_tensor * pos_q_intra, ggml_tensor * pos_q_succ, ggml_tensor * pos_q_inter, ++ ggml_tensor * mask_intra, ggml_tensor * mask_succ, ggml_tensor * mask_inter, ++ ggml_tensor * pos_q_real, ++ const llama_ubatch * ubatch, uint32_t chunk_size) const { ++ GGML_ASSERT(n_stream == 1 && "dca: single-stream only (1M context is single-sequence); TODO multi-stream"); ++ GGML_ASSERT(chunk_size >= 1); ++ ++ const auto & cells = v_cells[0]; ++ ++ const int64_t n_tokens = ubatch->n_tokens; ++ // opencoti F5 dca perf (T87.pD-gen #617): pos_k is no longer allocated (K is PRE-roped at insert, ++ // build_attn_dca_core no longer ropes the cache), so derive n_kv from the INTRA mask (always ++ // present, mask_intra->ne[0] == n_kv) and skip the pos_k fill. pos_k arrives nullptr; the ++ // parameter is retained only to keep the shared set_input_dca signature stable. ++ (void) pos_k; ++ const int64_t n_kv = mask_intra->ne[0]; ++ const int64_t c = (int64_t) chunk_size; ++ ++ GGML_ASSERT(ggml_backend_buffer_is_host(mask_intra->buffer)); ++ GGML_ASSERT(mask_intra->ne[1] >= n_tokens); ++ ++ // pos_q per regime. opencoti F5 dca perf (T87.pD-opt Stage 1): in single-chunk serving ++ // (n_ctx <= c) only INTRA is allocated/consumed; SUCC/INTER tensors are null here — guard every ++ // SUCC/INTER write so set_input never touches an unallocated input's null ->data. ++ int32_t * pqi = (int32_t *) pos_q_intra->data; ++ int32_t * pqs = pos_q_succ ? (int32_t *) pos_q_succ->data : nullptr; ++ int32_t * pqn = pos_q_inter ? (int32_t *) pos_q_inter->data : nullptr; ++ // opencoti F5 dca (#623): MROPE/IMROPE archs (qwen35/qwen35moe) allocate pos_q 4-wide (section ++ // split) so the per-regime Q ropes via ggml_rope_multi; NEOX archs keep it 1-wide. Derive the per- ++ // token id count from the tensor shape and write the remapped position into sections 0-2 with ++ // section 3 = 0 — the exact layout llm_graph_input_pos::set_input builds for native MROPE TEXT. ++ // npp==1 reduces to the previous scalar write byte-identically. pos_q_real stays scalar (band channel). ++ const int64_t npp = pos_q_intra->ne[0] / n_tokens; ++ // O6-fix: real abs query positions for the analytical band (kernel: cq = real_pos/chunk = the TRUE ++ // chunk index). Regime-independent; null unless the multi-chunk analytical path allocated it. The ++ // RoPE-remapped pqs (oq+c) / pqn (c const) would pin cq==1 for every query at chunk>=2 — SUCC band ++ // collapses to chunk 0, INTER band empties — so they cannot double as the band channel. ++ int32_t * pqr = pos_q_real ? (int32_t *) pos_q_real->data : nullptr; ++ for (int64_t i = 0; i < n_tokens; ++i) { ++ const llama_pos p1 = ubatch->pos[i]; ++ const int32_t oq = (int32_t) (p1 % c); ++ for (int64_t s = 0; s < npp; ++s) { ++ const int64_t off = s * n_tokens + i; // contiguous per-section blocks (native MROPE layout) ++ const bool spatial = (s < 3); // sections 0-2 carry the position; section 3 = 0 for text ++ pqi[off] = spatial ? oq : 0; ++ if (pqs) { pqs[off] = spatial ? (oq + (int32_t) c) : 0; } ++ // opencoti DCA #635 FIX: INTER-Q position = 2c, NOT c. K is pre-roped ONCE at j%c (#617), ++ // so the INTER relative distance is q_inter - (j%c). With q_inter = c that is c-(j%c) in ++ // [1,c], which OVERLAPS the recent/SUCC window (SUCC true rel runs up to 2c-1): a chunk-end ++ // far key lands at rel~1 and masquerades as the most-recent token, so at the FIRST cq>=2 ++ // query the older chunks flood ~50% of the attention mass and the model loses the thread ++ // (verified: count-up collapses exactly at pos==2c; true-ref relerr 1.1; INTER mass 0.50). ++ // q_inter = 2c gives rel = 2c-(j%c) in [c+1,2c]: every far key is strictly BEYOND the window ++ // edge c (canonical DCA places them AT c), so they stay retrievable but cannot out-compete ++ // genuine recent keys. Offline-validated on captured vectors: INTER mass 0.50 -> 0.06, ++ // recent-argmax 0.28-0.81 -> 0.91-1.00. (This is an approximation forced by #617's ++ // pre-roped-once K, which cannot realize canonical DCA's constant rel==c for INTER.) ++ if (pqn) { pqn[off] = spatial ? (int32_t)(2 * c) : 0; } ++ } ++ if (pqr) { pqr[i] = (int32_t) p1; } // analytical band channel: always scalar, never roped ++ } ++ ++ // masks: init -inf, then open the INTRA band for causal same-seq nonempty cells. ++ float * mi = (float *) mask_intra->data; ++ ++ // opencoti F5 dca (#617B): FRAGMENTED fallback. After a classic context-shift the cache is ++ // position-relabeled in place (cell index != absolute position, with holes), so the analytical ++ // chunk bands — which assume index == pos — select the WRONG physical cells and the model ++ // collapses. build_attn_inp_dca detects this via cells.get_is_fragmented() and allocates the dense ++ // SUCC/INTER masks (mask_succ/mask_inter non-null here ⇒ this branch). Fill all THREE regimes by ++ // reading each cell's ACTUAL position over the FULL [0,n_kv): for query chunk cq=p1/c and key ++ // chunk ck=p0/c — INTRA iff ck==cq, SUCC iff ck==cq-1, INTER iff ckbuffer)); ++ GGML_ASSERT(ggml_backend_buffer_is_host(mask_inter->buffer)); ++ float * ms = (float *) mask_succ->data; ++ float * mn = (float *) mask_inter->data; ++ const int64_t ne1 = mask_intra->ne[1]; ++ for (int64_t t = 0; t < n_kv*ne1; ++t) { mi[t] = -INFINITY; ms[t] = -INFINITY; mn[t] = -INFINITY; } ++ std::vector kpos0(n_kv); // cell position, or -1 for an empty/hole cell (skip) ++ for (int64_t j = 0; j < n_kv; ++j) { ++ kpos0[j] = cells.is_empty(j) ? -1 : (int32_t) cells.pos_get(j); ++ } ++ for (int64_t i = 0; i < n_tokens; ++i) { ++ const llama_seq_id seq_id = ubatch->seq_id[i][0]; ++ const llama_pos p1 = ubatch->pos[i]; ++ const int64_t cq = (int64_t) (p1 / c); ++ const uint64_t idst = (uint64_t) n_kv * i; ++ for (int64_t j = 0; j < n_kv; ++j) { ++ const int32_t p0 = kpos0[j]; ++ if (p0 < 0) { continue; } // empty / hole ++ if (p0 > p1) { continue; } // causal ++ if (!cells.seq_has(j, seq_id)) { continue; } // same seq ++ const int64_t ck = (int64_t) (p0 / c); // ck <= cq under causal p0 <= p1 ++ if (ck == cq) { mi[idst + j] = 0.0f; } // INTRA: same chunk ++ else if (ck == cq - 1) { ms[idst + j] = 0.0f; } // SUCC: immediately-prior chunk ++ else { mn[idst + j] = 0.0f; } // INTER: any older chunk (ck < cq-1) ++ } ++ } ++ return; ++ } ++ ++ // opencoti F5 dca perf (O9 analytical-INTRA host window). Under analytical_bands (signalled by a ++ // non-null pos_q_real; SUCC/INTER masks are null and derived on-GPU) the fused kernel reads the INTRA ++ // mask ONLY in the single-chunk index band [cq*c, (cq+1)*c) — its INTRA phase starts at ++ // intra_lo = cq*dca_c_blk (fattn-mma-f16.cuh) and cq is UNIFORM per ubatch (chunk_size % n_ubatch == 0). ++ // Every row below cq*c (5/6 of n_kv at 256k, 121/122 at 1M) is filled+copied today but NEVER read. ++ // Narrowing the fill to the chunk window makes the host mask cost O(c*n_tokens) — INDEPENDENT of n_kv — ++ // so DCA-on prefill stops starving the GPU as context grows. The SKIP-maskfill A/B proved the ++ // full-width fill was the ENTIRE residual 256k bottleneck (SM-util p10 58->100, 3403->4014 t/s ~= the ++ // native-off 4034). Only the j-RANGE is narrowed to provably-unread bounds; the per-cell causal ++ // (p0<=p1) + seq_has checks are byte-identical to the full path. Keys in [cq*c,(cq+1)*c) are all in ++ // chunk cq, so only the INTRA mask is written (no ck compare). single_chunk / legacy-3-mask paths ++ // (pos_q_real==null) keep the proven full-width fill below. ++ if (pos_q_real) { ++ const int64_t cq = (int64_t) (ubatch->pos[0] / c); // uniform per ubatch under analytical bands ++ const int64_t lo = cq * c; ++ const int64_t hi = (cq + 1) * c < n_kv ? (cq + 1) * c : n_kv; ++ std::vector wpos(hi > lo ? (size_t) (hi - lo) : 0); // window cell positions (empty => -1), reused per query ++ for (int64_t j = lo; j < hi; ++j) { ++ wpos[j - lo] = cells.is_empty(j) ? -1 : (int32_t) cells.pos_get(j); ++ } ++ for (int64_t i = 0; i < n_tokens; ++i) { ++ float * const row = mi + (uint64_t) n_kv * i; ++ for (int64_t j = lo; j < hi; ++j) { row[j] = -INFINITY; } // vectorizable window memset ++ const llama_seq_id seq_id = ubatch->seq_id[i][0]; ++ const llama_pos p1 = ubatch->pos[i]; ++ for (int64_t j = lo; j < hi; ++j) { ++ const int32_t p0 = wpos[j - lo]; ++ if (p0 < 0) { continue; } // empty cell ++ if (p0 > p1) { continue; } // causal ++ if (!cells.seq_has(j, seq_id)) { continue; } // per-query seq match (cheap bitset, as native) ++ row[j] = 0.0f; // window ⊂ chunk cq ⇒ INTRA ++ } ++ } ++ return; ++ } ++ ++ // opencoti F5 dca (#618): the legacy dense 3-regime (INTRA/SUCC/INTER) full-width fill was REMOVED. ++ // This path is reached ONLY for single_chunk serving (n_ctx <= chunk ⇒ pos_q_real == null and the ++ // SUCC/INTER masks unallocated): every query and key live in chunk 0, so the partition collapses to ++ // INTRA-only with no chunk-index compare. Fill the single native-causal INTRA mask. Keep the two ++ // passes (vectorizable -inf memset, then sparse causal open) — fusing them regressed prefill ~20% ++ // (O8, reverted): a per-cell store guarded by a conditional cannot vectorize. ++ const int64_t ne1 = mask_intra->ne[1]; ++ for (int64_t t = 0; t < n_kv*ne1; ++t) { ++ mi[t] = -INFINITY; ++ } ++ std::vector kpos0(n_kv); // cell position, or -1 for an empty cell (skip) ++ for (int64_t j = 0; j < n_kv; ++j) { ++ kpos0[j] = cells.is_empty(j) ? -1 : (int32_t) cells.pos_get(j); ++ } ++ for (int64_t i = 0; i < n_tokens; ++i) { ++ const llama_seq_id seq_id = ubatch->seq_id[i][0]; ++ const llama_pos p1 = ubatch->pos[i]; ++ const uint64_t idst = (uint64_t) n_kv * i; ++ for (int64_t j = 0; j < n_kv; ++j) { ++ const int32_t p0 = kpos0[j]; ++ if (p0 < 0) { continue; } // empty cell ++ if (p0 > p1) { continue; } // causal ++ if (!cells.seq_has(j, seq_id)) { continue; } // per-query seq match (cheap bitset, as native) ++ mi[idst + j] = 0.0f; // single chunk ⇒ every causal key is INTRA ++ } ++ } ++} ++ + void llama_kv_cache::set_input_k_rot(ggml_tensor * dst) const { + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + +@@ -3917,7 +4150,14 @@ public: + + void set_input(const llama_ubatch * ubatch) override; + +- ggml_tensor * k_shift; // I32 [kv_size*n_stream] ++ ggml_tensor * k_shift = nullptr; // I32 [kv_size*n_stream] ++ ++ // opencoti F5 dca (#593, T87.pD-gen #617): parallel shift channel for DCA global layers, which ++ // cache PRE-roped K (rope-at-(pos mod c)) and so need the per-cell MODULAR rotation delta ++ // (pos_new mod c) - (pos_old mod c), not the native absolute pos_new - pos_old. Null (and unread) ++ // unless this cache owns a DCA layer. dca_chunk = the resolved chunk size c. ++ ggml_tensor * k_shift_dca = nullptr; // I32 [kv_size*n_stream] ++ uint32_t dca_chunk = 0; + + // note: assumes k_rot^2 == I + ggml_tensor * k_rot = nullptr; +@@ -3932,6 +4172,10 @@ void llm_graph_input_k_shift::set_input(const llama_ubatch * ubatch) { + kv_self->set_input_k_shift(k_shift); + } + ++ if (k_shift_dca) { ++ kv_self->set_input_k_shift_dca(k_shift_dca, dca_chunk); ++ } ++ + if (k_rot) { + kv_self->set_input_k_rot(k_rot); + } +@@ -3943,12 +4187,41 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co + + auto inp = std::make_unique(this); + +- inp->k_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, (int64_t) get_size()*n_stream); +- ggml_set_input(inp->k_shift); ++ const auto & cparams = lctx->get_cparams(); ++ ++ // opencoti F5 dca (#593, T87.pD-gen #617): the native k_shift channel is consumed only by layers ++ // that take the native ABSOLUTE shift — SWA layers, or any layer when DCA is off. When DCA is on ++ // AND every layer is full-attention (e.g. Qwen: all layers route to k_shift_dca in the loop below), ++ // the native k_shift is an ORPHAN graph input: the allocator assigns it no buffer, yet set_input ++ // would still write to it -> GGML_ASSERT(buffer) at the first context shift. Allocate it ONLY when ++ // some layer actually uses it (mirrors the has_dca_layer guard for k_shift_dca). k_shift defaults ++ // to nullptr so set_input's `if (k_shift)` guard correctly skips the fill when it's not allocated. ++ bool needs_native_shift = false; ++ for (const auto & layer : layers) { ++ if (!(cparams.dca_enabled && !hparams.is_swa(layer.il))) { needs_native_shift = true; break; } ++ } ++ if (needs_native_shift) { ++ inp->k_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, (int64_t) get_size()*n_stream); ++ ggml_set_input(inp->k_shift); ++ } + + inp->k_rot = build_input_k_rot(ctx); + +- const auto & cparams = lctx->get_cparams(); ++ // opencoti F5 dca (#593, T87.pD-gen #617): DCA global layers cache PRE-roped K (rope-at-(pos mod ++ // c)), so their KV-shift needs a per-cell MODULAR delta, not the native absolute pos_new-pos_old. ++ // Allocate a parallel shift channel + record the chunk size c, but ONLY when this cache actually ++ // owns a DCA (non-SWA) layer — otherwise it'd be an orphan graph input (null buffer -> null write). ++ if (cparams.dca_enabled) { ++ bool has_dca_layer = false; ++ for (const auto & layer : layers) { ++ if (!hparams.is_swa(layer.il)) { has_dca_layer = true; break; } ++ } ++ if (has_dca_layer) { ++ inp->k_shift_dca = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, (int64_t) get_size()*n_stream); ++ ggml_set_input(inp->k_shift_dca); ++ inp->dca_chunk = dca_resolve_chunk_size(cparams, hparams); ++ } ++ } + + // opencoti F4 M3 Phase 4-C: k-shift loops over per-stream tensors. + // With per-stream owning tensors, each stream's K is a separate buffer, +@@ -3987,10 +4260,17 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co + ggml_row_size(layer_k->type, n_embd_k_gqa), + ggml_row_size(layer_k->type, n_embd_nope)); + ++ // opencoti F5 dca (#593, T87.pD-gen #617): DCA global layers store PRE-roped K, so they ++ // shift via the modular-delta channel (k_shift_dca); SWA / non-DCA layers stay on the ++ // native absolute k_shift. The DCA channel is only allocated when this cache owns a DCA ++ // layer — exactly when the predicate below can be true. ++ ggml_tensor * k_shift_src = (cparams.dca_enabled && !hparams.is_swa(il)) ++ ? inp->k_shift_dca ++ : inp->k_shift; + ggml_tensor * k_shift_s = (n_stream == 1) +- ? inp->k_shift +- : ggml_view_1d(ctx, inp->k_shift, get_size(), +- (size_t) s * get_size() * inp->k_shift->nb[0]); ++ ? k_shift_src ++ : ggml_view_1d(ctx, k_shift_src, get_size(), ++ (size_t) s * get_size() * k_shift_src->nb[0]); + + ggml_tensor * cur = build_rope_shift(cparams, ctx, k, k_shift_s, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); + +@@ -4803,6 +5083,10 @@ uint32_t llama_kv_cache_context::get_n_kv() const { + return n_kv; + } + ++bool llama_kv_cache_context::get_is_fragmented() const { ++ return kv->get_is_fragmented(); ++} ++ + ggml_type llama_kv_cache_context::type_k() const { + return kv->type_k(); + } +@@ -4919,6 +5203,17 @@ void llama_kv_cache_context::set_input_pos_bucket(ggml_tensor * dst, const llama + kv->set_input_pos_bucket(dst, ubatch); + } + ++// opencoti F5 dca (#593) — per-batch wrapper around the DCA host-fill. ++void llama_kv_cache_context::set_input_dca( ++ ggml_tensor * pos_k, ++ ggml_tensor * pos_q_intra, ggml_tensor * pos_q_succ, ggml_tensor * pos_q_inter, ++ ggml_tensor * mask_intra, ggml_tensor * mask_succ, ggml_tensor * mask_inter, ++ ggml_tensor * pos_q_real, ++ const llama_ubatch * ubatch, uint32_t chunk_size) const { ++ kv->set_input_dca(pos_k, pos_q_intra, pos_q_succ, pos_q_inter, ++ mask_intra, mask_succ, mask_inter, pos_q_real, ubatch, chunk_size); ++} ++ + void llama_kv_cache_context::set_input_k_rot(ggml_tensor * dst) const { + kv->set_input_k_rot(dst); + } +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -185,6 +185,11 @@ public: + + bool get_has_shift() const; + ++ // opencoti F5 dca (#617B): true if any stream's cells have been position-relabeled in place ++ // (context-shift / self-extend) ⇒ cell index != absolute position. Read at DCA graph-build time ++ // to gate the position-based dense-mask fallback. See llama_kv_cells::get_is_fragmented. ++ bool get_is_fragmented() const; ++ + // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md + // Idempotent. Zeroes cells [n_cells_cleared, up_to_cells) of every KV + // layer's K and V tensors across every stream. The buffers are malloc'd +@@ -307,9 +312,17 @@ public: + void set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const; + + void set_input_k_shift(ggml_tensor * dst) const; ++ // opencoti F5 dca (#593, T87.pD-gen #617) — modular K-shift delta for DCA global layers (which ++ // cache PRE-roped K at pos mod c). Fills dst[i] = (pos_new % c) - (pos_old % c) per cell. ++ void set_input_k_shift_dca(ggml_tensor * dst, uint32_t chunk_size) const; + + void set_input_kq_mask (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const; + void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const; ++ // opencoti F5 dca (#593) — DCA 3-regime position-remap host-fill (single-stream). ++ void set_input_dca(ggml_tensor * pos_k, ggml_tensor * pos_q_intra, ggml_tensor * pos_q_succ, ++ ggml_tensor * pos_q_inter, ggml_tensor * mask_intra, ggml_tensor * mask_succ, ++ ggml_tensor * mask_inter, ggml_tensor * pos_q_real, ++ const llama_ubatch * ubatch, uint32_t chunk_size) const; + + void set_input_k_rot(ggml_tensor * dst) const; + void set_input_v_rot(ggml_tensor * dst) const; +@@ -593,6 +606,11 @@ public: + + uint32_t get_n_kv() const; + ++ // opencoti F5 dca (#617B): delegate the cache's non-contiguity latch (index != pos after a ++ // context-shift / self-extend). build_attn_inp_dca reads this at graph-build to decide whether to ++ // allocate the dense SUCC/INTER fallback masks for this ubatch. ++ bool get_is_fragmented() const; ++ + ggml_type type_k() const; + ggml_type type_v() const; + +@@ -650,6 +668,11 @@ public: + void set_input_k_shift (ggml_tensor * dst) const; + void set_input_kq_mask (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const; + void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const; ++ // opencoti F5 dca (#593) — DCA 3-regime position-remap host-fill (single-stream). ++ void set_input_dca(ggml_tensor * pos_k, ggml_tensor * pos_q_intra, ggml_tensor * pos_q_succ, ++ ggml_tensor * pos_q_inter, ggml_tensor * mask_intra, ggml_tensor * mask_succ, ++ ggml_tensor * mask_inter, ggml_tensor * pos_q_real, ++ const llama_ubatch * ubatch, uint32_t chunk_size) const; + + void set_input_k_rot(ggml_tensor * dst) const; + void set_input_v_rot(ggml_tensor * dst) const; +diff --git a/llama.cpp/src/llama-kv-cells.h b/llama.cpp/src/llama-kv-cells.h +--- a/llama.cpp/src/llama-kv-cells.h ++++ b/llama.cpp/src/llama-kv-cells.h +@@ -41,6 +41,10 @@ public: + + has_shift = false; + ++ // opencoti F5 dca (#617B): the non-contiguity latch (index!=pos) clears ONLY on a full reset. ++ // reset_shift() must NOT touch it — see the member comment below. ++ is_fragmented = false; ++ + used.clear(); + + for (uint32_t s = 0; s < LLAMA_MAX_SEQ; ++s) { +@@ -96,6 +100,16 @@ public: + return has_shift; + } + ++ // opencoti F5 dca (#617B): true once any operation has relabeled a cell's position in place ++ // (pos_add / pos_div, i.e. a classic context-shift or self-extend), breaking the cell-index == ++ // absolute-position invariant that DCA's analytical band selection depends on. STICKY across ++ // reset_shift() (which only clears the transient rope-delta queue, leaving pos[] relabeled) — ++ // cleared ONLY on a full reset(). The DCA host-fill gates a position-based dense-mask fallback on ++ // this so a shifted (non-contiguous) cache stays correct; a never-shifted cache keeps the fast path. ++ bool get_is_fragmented() const { ++ return is_fragmented; ++ } ++ + // move cell isrc to idst (used during defrag) + //void mv(uint32_t isrc, uint32_t idst) { + // assert(isrc < pos.size()); +@@ -420,6 +434,7 @@ public: + shift[i] += d; + + has_shift = true; ++ is_fragmented = true; // opencoti F5 dca (#617B): pos[i] relabeled in place ⇒ index != pos + + if (pos[i] < 0) { + seq[i].reset(); +@@ -453,11 +468,20 @@ public: + seq_pos_add(i); + + has_shift = true; ++ is_fragmented = true; // opencoti F5 dca (#617B): pos[i] relabeled in place ⇒ index != pos + } + + private: + bool has_shift = false; + ++ // opencoti F5 dca (#617B): sticky "cell index != absolute position somewhere" latch. Set true by ++ // pos_add / pos_div (the only ops that relabel a live cell's position in place — context-shift's ++ // seq_add and self-extend's seq_div); NOT cleared by reset_shift() (positions stay relabeled after ++ // the rope-shift applies), only by reset(). Plain rm / seq_rm are NOT flagged: a removal leaves ++ // surviving cells at pos==index (holes are just empty slots the dense mask skips), and prompt-cache ++ // reuse calls seq_rm every request — flagging it would needlessly force the slow path on the hot flow. ++ bool is_fragmented = false; ++ + // set of indices of used cells (i.e. pos[i] != -1, allowed to not have any seq_id) + std::set used; + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -1747,6 +1747,7 @@ struct llama_model_qwen35 : public llama_model_base { + private: + ggml_tensor * build_layer_attn( + llm_graph_input_attn_kv * inp_attn, ++ llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off, unchanged) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +@@ -1793,6 +1794,7 @@ struct llama_model_qwen35moe : public llama_model_base { + private: + ggml_tensor * build_layer_attn( + llm_graph_input_attn_kv * inp_attn, ++ llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off, unchanged) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +diff --git a/llama.cpp/src/models/qwen2.cpp b/llama.cpp/src/models/qwen2.cpp +--- a/llama.cpp/src/models/qwen2.cpp ++++ b/llama.cpp/src/models/qwen2.cpp +@@ -1,4 +1,5 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca #593 + + void llama_model_qwen2::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); +@@ -65,6 +66,11 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param + + auto * inp_attn = build_attn_inp_kv(); + ++ // opencoti F5 dca #593 — Qwen: every layer is full attention, so DCA applies to all of ++ // them. Build the 3-regime position remap once on the plain-kv context. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx) : nullptr; ++ + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { +@@ -82,25 +88,35 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope Q; rope per regime in build_attn_dca + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope K + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + ++ if (use_dca) { // opencoti F5 dca #593 ++ const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ } else { + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ } + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); +diff --git a/llama.cpp/src/models/qwen3.cpp b/llama.cpp/src/models/qwen3.cpp +--- a/llama.cpp/src/models/qwen3.cpp ++++ b/llama.cpp/src/models/qwen3.cpp +@@ -1,4 +1,5 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca #593 + + void llama_model_qwen3::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); +@@ -65,6 +66,11 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param + + auto * inp_attn = build_attn_inp_kv(); + ++ // opencoti F5 dca #593 — Qwen: every layer is full attention, so DCA applies to all of ++ // them. Build the 3-regime position remap once on the plain-kv context. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx) : nullptr; ++ + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { +@@ -85,28 +91,38 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope Q; rope per regime in build_attn_dca + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope K + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + ++ if (use_dca) { // opencoti F5 dca #593 ++ const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ } else { + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ } + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); +diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp +--- a/llama.cpp/src/models/qwen35.cpp ++++ b/llama.cpp/src/models/qwen35.cpp +@@ -1,4 +1,5 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca #623 — qwen35 (Qwen3.5-Next hybrid) DCA on the full-attn layers + #include "llama-memory-recurrent.h" + + void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) { +@@ -154,6 +155,12 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + + auto * inp = build_inp_mem_hybrid(); + ++ // opencoti F5 dca #623 — DCA applies ONLY to the full-attention layers (is_recurrent==false); ++ // the gated-delta-net (SSM) layers are recurrent and never chunked. Build the 3-regime position ++ // remap once on the attn kv-context. inp_dca==nullptr (default --dca off) => unchanged behaviour. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp->get_attn()->mctx) : nullptr; ++ + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +@@ -173,7 +180,7 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + cur = build_layer_attn_linear(inp->get_recr(), cur, il); + } else { + // Full attention layer +- cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); ++ cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il); + } + + if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { +@@ -259,6 +266,7 @@ ggml_tensor * llama_model_qwen35::graph::build_norm_gated( + + ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + llm_graph_input_attn_kv * inp, ++ llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +@@ -301,29 +309,41 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + +- // Apply MRoPE +- Qcur = ggml_rope_multi( +- ctx0, Qcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow +- ); +- +- Kcur = ggml_rope_multi( +- ctx0, Kcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow +- ); +- +- cb(Qcur, "Qcur", il); +- cb(Kcur, "Kcur", il); +- cb(Vcur, "Vcur", il); +- + // Attention computation + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + +- cur = build_attn(inp, +- nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ if (inp_dca) { ++ // opencoti F5 dca #623 — DCA path: cache PRE-rope Q/K and rope per-regime INSIDE build_attn_dca ++ // (so SKIP the MRoPE here). wo=nullptr returns the pre-projection attention output [n_embd, ++ // n_tokens] (build_attn_dca_core skips the proj on null wo) — exactly like the stock gated path, ++ // so the gate_sigmoid multiply + wo below run unchanged. Pass the model's IMRoPE section split so ++ // build_dca_rope ropes the DCA-remapped position via ggml_rope_multi with 4-wide positions — the ++ // NEOX-collapse produced garbage (gate diverged @char 21, #623). DCA needs an f16 K cache (default). ++ const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr, ++ { sections[0], sections[1], sections[2], sections[3] } }; ++ cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il); ++ } else { ++ // Apply MRoPE ++ Qcur = ggml_rope_multi( ++ ctx0, Qcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow ++ ); ++ ++ Kcur = ggml_rope_multi( ++ ctx0, Kcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow ++ ); ++ ++ cb(Qcur, "Qcur", il); ++ cb(Kcur, "Kcur", il); ++ cb(Vcur, "Vcur", il); ++ ++ cur = build_attn(inp, ++ nullptr, nullptr, nullptr, ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ } + cb(cur, "attn_pregate", il); + + ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate); +diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp +--- a/llama.cpp/src/models/qwen35moe.cpp ++++ b/llama.cpp/src/models/qwen35moe.cpp +@@ -1,4 +1,5 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca #623 — qwen35moe (Qwen3.5-Next hybrid MoE) DCA on full-attn layers + #include "llama-memory-recurrent.h" + + void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) { +@@ -177,6 +178,12 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p + + auto * inp = build_inp_mem_hybrid(); + ++ // opencoti F5 dca #623 — DCA applies ONLY to the full-attention layers (is_recurrent==false); ++ // the gated-delta-net (SSM) layers are recurrent and never chunked. Build the 3-regime position ++ // remap once on the attn kv-context. inp_dca==nullptr (default --dca off) => unchanged behaviour. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp->get_attn()->mctx) : nullptr; ++ + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +@@ -196,7 +203,7 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p + cur = build_layer_attn_linear(inp->get_recr(), cur, il); + } else { + // Full attention layer +- cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); ++ cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il); + } + + if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { +@@ -282,6 +289,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_norm_gated( + + ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + llm_graph_input_attn_kv * inp, ++ llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +@@ -324,29 +332,41 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + +- // Apply IMRoPE +- Qcur = ggml_rope_multi( +- ctx0, Qcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow +- ); +- +- Kcur = ggml_rope_multi( +- ctx0, Kcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow +- ); +- +- cb(Qcur, "Qcur", il); +- cb(Kcur, "Kcur", il); +- cb(Vcur, "Vcur", il); +- + // Attention computation + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + +- cur = build_attn(inp, +- nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ if (inp_dca) { ++ // opencoti F5 dca #623 — DCA path: cache PRE-rope Q/K and rope per-regime INSIDE build_attn_dca ++ // (so SKIP the IMRoPE here). wo=nullptr returns the pre-projection attention output [n_embd, ++ // n_tokens] (build_attn_dca_core skips the proj on null wo) — exactly like the stock gated path, ++ // so the gate_sigmoid multiply + wo below run unchanged. Pass the model's IMRoPE section split so ++ // build_dca_rope ropes the DCA-remapped position via ggml_rope_multi with 4-wide positions — the ++ // NEOX-collapse produced garbage (gate diverged @char 21, #623). DCA needs an f16 K cache (default). ++ const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr, ++ { sections[0], sections[1], sections[2], sections[3] } }; ++ cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il); ++ } else { ++ // Apply IMRoPE ++ Qcur = ggml_rope_multi( ++ ctx0, Qcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow ++ ); ++ ++ Kcur = ggml_rope_multi( ++ ctx0, Kcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow ++ ); ++ ++ cb(Qcur, "Qcur", il); ++ cb(Kcur, "Kcur", il); ++ cb(Vcur, "Vcur", il); ++ ++ cur = build_attn(inp, ++ nullptr, nullptr, nullptr, ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ } + cb(cur, "attn_pregate", il); + + ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate); +diff --git a/llama.cpp/src/models/qwen3moe.cpp b/llama.cpp/src/models/qwen3moe.cpp +--- a/llama.cpp/src/models/qwen3moe.cpp ++++ b/llama.cpp/src/models/qwen3moe.cpp +@@ -1,4 +1,5 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca #593 + + void llama_model_qwen3moe::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); +@@ -75,6 +76,11 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa + + auto * inp_attn = build_attn_inp_kv(); + ++ // opencoti F5 dca #593 — Qwen: every layer is full attention, so DCA applies to all of ++ // them. Build the 3-regime position remap once on the plain-kv context. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx) : nullptr; ++ + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { +@@ -95,28 +101,38 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope Q; rope per regime in build_attn_dca + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + ++ if (!use_dca) { // opencoti F5 dca #593 — cache PRE-rope K + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); ++ } + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + ++ if (use_dca) { // opencoti F5 dca #593 ++ const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ } else { + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ } + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -889,6 +889,17 @@ private: + return false; + } + ++ // opencoti F5 dca (#618): a context-init failure (e.g. the DCA `chunk %% n_ubatch == 0` ++ // precondition, or the pre-existing `n_ctx_seq == 0` / OOM / control-vector paths) makes ++ // common_init_from_params return with a loaded model but a NULL context. Without this guard ++ // the very next line (llama_n_ctx(ctx_tgt)) dereferences null -> SIGSEGV/coredump, masking the ++ // clear init error logged above. Mirror the model-null check so any such failure exits ++ // cleanly (false) with the error already on the log, instead of crashing. ++ if (ctx_tgt == nullptr) { ++ SRV_ERR("failed to create context (see initialization error above), '%s'\n", params_base.model.path.c_str()); ++ return false; ++ } ++ + vocab = llama_model_get_vocab(model_tgt); + + n_ctx = llama_n_ctx(ctx_tgt); +@@ -1019,8 +1030,18 @@ private: + + int n_ctx_slot = llama_n_ctx_seq(ctx_tgt); + if (n_ctx_slot > n_ctx_train) { +- SRV_WRN("the slot context (%d) exceeds the training context of the model (%d) - capping\n", n_ctx_slot, n_ctx_train); +- n_ctx_slot = n_ctx_train; ++ // opencoti F5 dca (T87.pD): Dual Chunk Attention is a TRAINING-FREE long-context method — ++ // it remaps query/key positions so no rope position ever exceeds the chunk size (<= the ++ // pretrain window), letting the model serve sequences far beyond n_ctx_train. The stock ++ // guard below caps the slot at n_ctx_train (correct for vanilla models, where positions past ++ // train produce garbage), but that cap is exactly wrong under DCA and silently blocked the ++ // >256k path. When --dca is on, keep the full requested slot context. ++ if (params_base.dca_enabled) { ++ SRV_INF("the slot context (%d) exceeds the training context (%d) — allowed: DCA extends context\n", n_ctx_slot, n_ctx_train); ++ } else { ++ SRV_WRN("the slot context (%d) exceeds the training context of the model (%d) - capping\n", n_ctx_slot, n_ctx_train); ++ n_ctx_slot = n_ctx_train; ++ } + } + + slots.clear(); diff --git a/patches/0079-gemma-assistant-mtp.patch b/patches/0079-gemma-assistant-mtp.patch new file mode 100644 index 0000000000000000000000000000000000000000..6366e4ad69c9748e343ef1188dced52f9b85cdf6 --- /dev/null +++ b/patches/0079-gemma-assistant-mtp.patch @@ -0,0 +1,3013 @@ +From: opencoti +Subject: [PATCH 0079] Gemma 4 assistant-drafter MTP — restore onto 0.10.3, coexist with native NextN (#423) + +Restores the Gemma-4 A4B speculative MTP assistant drafter (the gemma4-assistant sub-model, +loaded INTO a gemma4 target via llama_model_load_mtp_from_file) on the llamafile 0.10.3 base, +as a first-class `--spec-type draft-assistant` that COEXISTS with the upstream-native NextN +MTP (`--spec-type draft-mtp`). Distinct enums (COMMON_SPECULATIVE_TYPE_MTP vs +COMMON_SPECULATIVE_TYPE_DRAFT_MTP) and graph types (LLM_GRAPH_TYPE_MTP vs +LLM_GRAPH_TYPE_DECODER_MTP) — no collision. + +The drafter cross-reads the target KV read-only at decode time (build_attn_mtp) and decodes on +ctx_tgt via llama_decode_mtp; no second llama_context / KV cache. It rides the 0.10.3 seq-aware +speculative framework (process/draft/accept/need_embd hooks replace the old server h_idx/seq_id +plumbing, so it composes with PolyKV / multi-seq for free) and with the turbo / rolling-KV / DCA +features below it in the stack. + +Key 0.10.3 adaptations vs the original 0074 capture: +- per-model class (llama_model_gemma4_assistant) via llama_model_mapping, not the obsolete + 0.10.1 giant-switch loader; +- llm_graph_input_attn_kv_iswa::set_input DECOUPLES the kq_mask setter from the self_k_idxs + buffer guard. 0.10.3 nested them, which would silently starve the read-only MTP draft graph's + live mask when its pruned KV-write idxs (null buffer) skip the whole base block — a silent + re-introduction of bug-404. Each idxs/mask/rot setter now guards on its own buffer; + byte-identical for every non-MTP graph. +- server loader gains a leading `has_dft() && assistant` branch that loads the assistant into + the target and leaves model_dft/ctx_dft null (null-safe across every ctx_dft checkpoint path, + which all early-return on a null context). + +opencoti upstream-alignment (#476 S1; see docs/protocols/UPSTREAM_SYNC.md): this patch now carries +the upstream llama.cpp #23398/#24282 DATA-PLANE identity for the assistant drafter — arch string +`gemma4-assistant` (hyphen), tensors `nextn.pre_projection`/`nextn.post_projection` + +`masked_embd_centroids`/`masked_embd_ordering`, and the generic `embedding_length_out` / +`nextn_predict_layers` KV — plus the matching internal C++ symbols (LLM_TENSOR_NEXTN_PROJ_*, +LLM_TENSOR_MASKED_EMBD_*, hparams n_embd_out_impl). The CONTROL-PLANE is deliberately KEPT +opencoti-native (single-context build_attn_mtp cross-read — NOT upstream's ctx_other second context, +which assumes vanilla single-owner unquantized KV and breaks PolyKV/rolling-KV/turbo/DCA; ordered +masked-embedder head; tok_embd→OUTPUT buft; fused N-step; `--spec-type draft-assistant`). The rename +is semantically INERT — proven by real_frac=0 logit-equivalence on the re-formatted A4B drafter. + +opencoti upstream-alignment (#476 S1; see docs/protocols/UPSTREAM_SYNC.md): this patch now carries +the upstream llama.cpp #23398/#24282 DATA-PLANE identity for the assistant drafter — arch string +`gemma4-assistant` (hyphen), tensors `nextn.pre_projection`/`nextn.post_projection` + +`masked_embd_centroids`/`masked_embd_ordering`, and the generic `embedding_length_out` / +`nextn_predict_layers` KV — plus the matching internal C++ symbols (LLM_TENSOR_NEXTN_PROJ_*, +LLM_TENSOR_MASKED_EMBD_*, hparams n_embd_out_impl). The CONTROL-PLANE is deliberately KEPT +opencoti-native (single-context build_attn_mtp cross-read — NOT upstream's ctx_other second context, +which assumes vanilla single-owner unquantized KV and breaks PolyKV/rolling-KV/turbo/DCA; ordered +masked-embedder head; tok_embd→OUTPUT buft; fused N-step; `--spec-type draft-assistant`). The rename +is semantically INERT — proven by real_frac=0 logit-equivalence on the re-formatted A4B drafter. + +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -109,6 +109,7 @@ + llama.cpp/src/models/gemma3.cpp \ + llama.cpp/src/models/gemma3n.cpp \ + llama.cpp/src/models/gemma4.cpp \ ++ llama.cpp/src/models/gemma4-assistant.cpp \ + llama.cpp/src/models/glm-dsa.cpp \ + llama.cpp/src/models/glm4-moe.cpp \ + llama.cpp/src/models/glm4.cpp \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -3816,6 +3816,13 @@ + params.speculative.draft.mparams.path = value; + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); ++ add_opt(common_arg( // opencoti F5 M6-S4 mtp — Gemma 4 assistant drafter ++ {"--mtp-head"}, "FNAME", ++ "alias for Gemma 4 MTP: path to gemma4_assistant GGUF (loaded into the target; use with --spec-type draft-assistant)", ++ [](common_params & params, const std::string & value) { ++ params.speculative.draft.mparams.path = value; ++ } ++ ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_MTP_HEAD")); + add_opt(common_arg( + {"--spec-type"}, common_speculative_all_types_str(), + string_format("comma-separated list of types of speculative decoding to use (default: %s)\n", +@@ -3826,6 +3833,16 @@ + params.speculative.types.insert(params.speculative.types.end(), types.begin(), types.end()); + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE")); ++ add_opt(common_arg( // opencoti F5 M6-S4 mtp ++ {"--draft-block-size"}, "N", ++ string_format("MTP draft block size B (drafts B-1 tokens per round; default: %d)", params.speculative.draft.draft_block_size), ++ [](common_params & params, int value) { ++ if (value < 2 || value > 32) { ++ throw std::invalid_argument("draft block size must be between 2 and 32"); ++ } ++ params.speculative.draft.draft_block_size = value; ++ } ++ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_DRAFT_BLOCK_SIZE")); + add_opt(common_arg( + {"--spec-ngram-mod-n-min"}, "N", + string_format("minimum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_min), +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -166,6 +166,7 @@ + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values + COMMON_SPECULATIVE_TYPE_NGRAM_MOD, + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache ++ COMMON_SPECULATIVE_TYPE_MTP, // opencoti F5 M6-S4 — Gemma 4 MTP assistant drafter + COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type + }; + +@@ -302,6 +303,10 @@ + int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding + int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding + ++ // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): draft block size B produces ++ // B-1 draft tokens per round. Default 3 (= 2 chained MTP draft steps). ++ int32_t draft_block_size = 3; ++ + float p_split = 0.1f; // speculative decoding split probability + float p_min = 0.0f; // minimum speculative decoding probability (greedy) + +diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp +--- a/llama.cpp/common/speculative.cpp ++++ b/llama.cpp/common/speculative.cpp +@@ -16,6 +16,11 @@ + #include + #include + #include ++#include // opencoti F5 M6-S4 mtp: std::sqrt (compute_h_l2) ++#include // opencoti F5 M6-S4 mtp: FILE/fopen/fputs (mtp_acc_tracer) ++#include // opencoti F5 M6-S4 mtp: std::getenv/std::atoi ++#include // opencoti F5 M6-S4 mtp: std::mutex (mtp_acc_tracer) ++#include // opencoti F5 M6-S4 mtp: std::ostringstream (trace_emit_*) + + #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 + #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 +@@ -29,7 +34,8 @@ + {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, + {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, + {"ngram-mod", COMMON_SPECULATIVE_TYPE_NGRAM_MOD}, +- {"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE} ++ {"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}, ++ {"draft-assistant", COMMON_SPECULATIVE_TYPE_MTP} // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter (distinct from native draft-mtp/NextN) + }; + + static std::string common_speculative_get_devices_str(const std::vector & devices) { +@@ -52,6 +58,63 @@ + const common_params_speculative & p = common_params_speculative{}) : type(t), params(p) {} + }; + ++// opencoti F5 M6-S4 mtp: target must be gemma4, assistant must be gemma4_assistant (loaded into target). ++// The fork uses a fork-only llama_model_arch_str() API; OURS reads the "general.architecture" GGUF key ++// via the existing public llama_model_meta_val_str — that key is exactly what the loader used to select ++// the arch enum, so it returns "gemma4" / "gemma4-assistant" canonically. (No new C API — P3 is host-only.) ++static bool common_speculative_mtp_arch_ok(const llama_model * model_tgt, const llama_model * model_dft) { ++ char arch_tgt[64] = {0}; ++ char arch_dft[64] = {0}; ++ llama_model_meta_val_str(model_tgt, "general.architecture", arch_tgt, sizeof(arch_tgt)); ++ llama_model_meta_val_str(model_dft, "general.architecture", arch_dft, sizeof(arch_dft)); ++ return std::strcmp(arch_tgt, "gemma4") == 0 ++ && std::strcmp(arch_dft, "gemma4-assistant") == 0; ++} ++ ++// opencoti F5 M6-S4 mtp: MTP-specific vocab compatibility. The assistant predicts the next token id ++// from the SAME SentencePiece vocab; stop/chat special tokens are owned by the target. So we require ++// identical vocab_type, size (within tolerance) and per-token text equality, and intentionally skip ++// bos/eos id + add_bos/add_eos checks (target may be chat-tuned eos==106, draft eos==1). ++static bool common_speculative_are_compatible_mtp( ++ const llama_model * model_tgt, ++ const llama_model * model_dft) { ++ const llama_vocab * vocab_tgt = llama_model_get_vocab(model_tgt); ++ const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); ++ ++ // opencoti F5 M6-S4 mtp: the gemma4_assistant draft head carries NO tokenizer (vocab type NONE); ++ // it shares the target's vocab, and load-time llama_model_load_mtp_from_file already verified ++ // n_tokens-equality. A NONE vocab has no token text to compare and is compatible by construction — ++ // accept here rather than failing the type/text checks below (which assume the draft has a real ++ // vocab; NONE != the target's real type would otherwise reject the assistant outright). ++ if (llama_vocab_type(vocab_dft) == LLAMA_VOCAB_TYPE_NONE) { ++ return true; ++ } ++ ++ if (llama_vocab_type(vocab_tgt) != llama_vocab_type(vocab_dft)) { ++ return false; ++ } ++ ++ const int n_vocab_tgt = llama_vocab_n_tokens(vocab_tgt); ++ const int n_vocab_dft = llama_vocab_n_tokens(vocab_dft); ++ const int vocab_diff = n_vocab_tgt > n_vocab_dft ++ ? n_vocab_tgt - n_vocab_dft ++ : n_vocab_dft - n_vocab_tgt; ++ ++ if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { ++ return false; ++ } ++ ++ for (int i = SPEC_VOCAB_CHECK_START_TOKEN_ID; i < std::min(n_vocab_tgt, n_vocab_dft); ++i) { ++ const char * t_tgt = llama_vocab_get_text(vocab_tgt, i); ++ const char * t_dft = llama_vocab_get_text(vocab_dft, i); ++ if (std::strcmp(t_tgt, t_dft) != 0) { ++ return false; ++ } ++ } ++ ++ return true; ++} ++ + static bool common_speculative_are_compatible( + const llama_model * model_tgt, + const llama_model * model_dft) { +@@ -1206,6 +1269,311 @@ + } + }; + ++// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. ++// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, ++// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV ++// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens ++// of a block in ONE facade call (no per-step sampler loop). ++// ++// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), ++// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never ++// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() ++// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) ++// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). ++ ++// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. ++// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. ++namespace { ++struct mtp_acc_tracer { ++ bool enabled = false; ++ FILE * fp = nullptr; ++ std::mutex mu; ++ ++ mtp_acc_tracer() { ++ const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); ++ if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { ++ return; ++ } ++ fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); ++ enabled = (fp != nullptr); ++ } ++ ++ ~mtp_acc_tracer() { ++ if (fp && fp != stderr) { ++ std::fclose(fp); ++ } ++ } ++ ++ void writeln(const std::string & line) { ++ if (!enabled) { ++ return; ++ } ++ std::lock_guard lk(mu); ++ std::fputs(line.c_str(), fp); ++ std::fputc('\n', fp); ++ std::fflush(fp); ++ } ++}; ++ ++mtp_acc_tracer & mtp_tracer() { ++ static mtp_acc_tracer t; ++ return t; ++} ++} // namespace ++ ++struct common_speculative_impl_draft_assistant : public common_speculative_impl { ++ common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) ++ ++ llama_context * ctx_tgt = nullptr; ++ ++ int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes ++ int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row ++ ++ // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows ++ // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds ++ // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h ++ // scheme so multi-seq (PolyKV) works without per-seq server plumbing. ++ std::vector> pending_h; // [n_seq][n_embd_backbone] ++ std::vector> verify_h; // [n_seq][n_rows * n_embd_backbone] ++ std::vector verify_h_rows; ++ std::vector i_batch_beg; ++ std::vector i_batch_end; ++ ++ // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head ++ // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain ++ // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. ++ int skip_streak_threshold = 0; ++ std::vector zero_accept_streak; ++ std::vector skip_last_draft; ++ std::vector prev_n_acc_at_draft; ++ ++ int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change ++ ++ static float compute_h_l2(const float * h, int32_t n) { ++ if (!h || n <= 0) { ++ return 0.0f; ++ } ++ double s = 0.0; ++ for (int32_t i = 0; i < n; ++i) { ++ s += (double) h[i] * (double) h[i]; ++ } ++ return (float) std::sqrt(s); ++ } ++ ++ common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) ++ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) ++ , params(params.draft) ++ { ++ ctx_tgt = this->params.ctx_tgt; ++ GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); ++ ++ const llama_model * model_tgt = llama_get_model(ctx_tgt); ++ n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); ++ n_embd_out = llama_model_n_embd_out(model_tgt); ++ ++ LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); ++ LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, ++ this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); ++ ++ // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. ++ llama_set_embeddings(ctx_tgt, true); ++ ++ if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { ++ const int v = std::atoi(e); ++ if (v >= 1 && v <= 32) { ++ skip_streak_threshold = v; ++ } ++ } ++ ++ pending_h.assign(n_seq, std::vector((size_t) std::max(1, n_embd_backbone), 0.0f)); ++ verify_h.assign(n_seq, {}); ++ verify_h_rows.assign(n_seq, 0); ++ i_batch_beg.assign(n_seq, -1); ++ i_batch_end.assign(n_seq, -1); ++ zero_accept_streak.assign(n_seq, 0); ++ skip_last_draft.assign(n_seq, 0); ++ prev_n_acc_at_draft.assign(n_seq, 0); ++ } ++ ++ ~common_speculative_impl_draft_assistant() override = default; ++ ++ void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { ++ llama_set_embeddings(ctx_tgt, true); ++ if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { ++ skip_last_draft[seq_id] = 0; ++ zero_accept_streak[seq_id] = 0; ++ } ++ } ++ ++ // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() ++ // can select the last-accepted token's hidden state. The spec framework calls process() after the ++ // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which ++ // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. ++ bool process(const llama_batch & batch_in) override { ++ if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { ++ return true; ++ } ++ if (n_embd_backbone <= 0) { ++ return true; ++ } ++ ++ const int32_t n_tokens = batch_in.n_tokens; ++ std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); ++ std::fill(i_batch_end.begin(), i_batch_end.end(), -1); ++ ++ for (int k = 0; k < n_tokens; ++k) { ++ const llama_seq_id s = batch_in.seq_id[k][0]; ++ if (s < 0 || s >= (llama_seq_id) n_seq) { ++ continue; ++ } ++ i_batch_end[s] = k; ++ if (i_batch_beg[s] < 0) { ++ i_batch_beg[s] = k; ++ } ++ } ++ ++ const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); ++ const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); ++ ++ for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { ++ if (i_batch_beg[s] < 0) { ++ continue; ++ } ++ const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; ++ verify_h_rows[s] = n_rows; ++ verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); ++ ++ for (int32_t i = 0; i < n_rows; ++i) { ++ const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); ++ if (h) { ++ std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, ++ (size_t) n_copy * sizeof(float)); ++ } ++ } ++ // default pending_h to the last row (correct when all drafts accepted / on prefill) ++ std::memcpy(pending_h[s].data(), ++ verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); ++ } ++ ++ return true; ++ } ++ ++ void draft(common_speculative_draft_params_vec & dparams) override { ++ if (n_embd_backbone <= 0) { ++ return; ++ } ++ ++ const int32_t block = params.draft_block_size; ++ const int32_t n_steps_raw = block > 1 ? block - 1 : 0; ++ ++ for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { ++ auto & dp = dparams[seq_id]; ++ if (!dp.drafting) { ++ continue; ++ } ++ ++ // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, ++ // the previous batch produced 0 accepted drafts. ++ if (skip_last_draft[seq_id]) { ++ skip_last_draft[seq_id] = 0; ++ } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { ++ zero_accept_streak[seq_id]++; ++ } else { ++ zero_accept_streak[seq_id] = 0; ++ } ++ if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { ++ zero_accept_streak[seq_id] = 0; ++ skip_last_draft[seq_id] = 1; ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ continue; // empty result -> server falls back to single-token verify ++ } ++ ++ int32_t n_steps = n_steps_raw; ++ if (dp.n_max > 0) { ++ n_steps = std::min(n_steps, dp.n_max); ++ } ++ n_steps = std::min(n_steps, params.n_max); ++ if (n_steps <= 0) { ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ continue; ++ } ++ ++ float * h_prev = pending_h[seq_id].data(); ++ ++ llama_memory_t mem = llama_get_memory(ctx_tgt); ++ llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; ++ if (attn_pos < 0) { ++ attn_pos = 0; ++ } ++ ++ std::vector out((size_t) n_steps, 0); ++ const int32_t rc = llama_decode_mtp( ++ ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, ++ /*out_drafts =*/ out.data(), ++ /*out_logits =*/ nullptr, ++ /*out_h_prev_last=*/ nullptr); ++ ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ ++ if (rc != 0) { ++ LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); ++ continue; ++ } ++ ++ auto & result = *dp.result; ++ for (int32_t i = 0; i < n_steps; ++i) { ++ result.push_back(out[i]); ++ } ++ ++ if (mtp_tracer().enabled) { ++ std::ostringstream oss; ++ oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter ++ << ",\"seq_id\":" << (int) seq_id ++ << ",\"id_last\":" << (int) dp.id_last ++ << ",\"attn_pos\":" << (int) attn_pos ++ << ",\"n_steps\":" << n_steps ++ << ",\"h_l2\":" << std::fixed << std::setprecision(4) ++ << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; ++ for (int32_t i = 0; i < n_steps; ++i) { ++ if (i) oss << ','; ++ oss << (int) out[i]; ++ } ++ oss << "]}"; ++ mtp_tracer().writeln(oss.str()); ++ } ++ } ++ } ++ ++ void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { ++ if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { ++ return; ++ } ++ const int32_t n_rows = verify_h_rows[seq_id]; ++ if (n_rows <= 0 || n_embd_backbone <= 0) { ++ return; ++ } ++ // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the ++ // last-accepted token's hidden state (clamped to the harvested rows). ++ const int32_t i_h = std::min(n_accepted, n_rows - 1); ++ std::memcpy(pending_h[seq_id].data(), ++ verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, ++ (size_t) n_embd_backbone * sizeof(float)); ++ ++ if (mtp_tracer().enabled) { ++ std::ostringstream oss; ++ oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter ++ << ",\"seq_id\":" << (int) seq_id ++ << ",\"n_accepted\":" << (int) n_accepted << "}"; ++ mtp_tracer().writeln(oss.str()); ++ ++trace_iter; ++ } ++ } ++ ++ bool need_embd() const override { ++ return true; // MTP reads POST-norm output embeddings (h_prev) ++ } ++}; ++ + struct common_speculative { + common_speculative_draft_params_vec dparams; + +@@ -1278,6 +1646,7 @@ + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: return "ngram-mod"; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram-cache"; ++ case COMMON_SPECULATIVE_TYPE_MTP: return "draft-assistant"; // opencoti F5 M6-S4 mtp + default: return "unknown"; + } + } +@@ -1330,6 +1699,9 @@ + bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); + bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 + bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; ++ // opencoti F5 M6-S4 mtp: our Gemma-4 assistant drafter (distinct from native NextN _DRAFT_MTP). ++ // Loaded INTO the target (no ctx_dft); validated against the in-target assistant below. ++ bool has_mtp_assistant = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_MTP)) != 0; + + bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); + bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); +@@ -1338,7 +1710,7 @@ + bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); + + // when adding a new type - update here the logic above +- static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); ++ static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); // opencoti F5 M6-S4 mtp: +_MTP (Gemma-4 assistant drafter) + + // this list here defines the priority of the speculators + // the one with highest priority are listed first +@@ -1364,7 +1736,11 @@ + LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__); + has_draft_simple = false; + } +- } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) { ++ } else if (has_draft_model_path && !has_mtp && !has_mtp_assistant && !has_draft_eagle3) { ++ // opencoti F5 M6-S4 mtp: do NOT auto-fallback to draft-simple when our Gemma-4 assistant ++ // is the draft type. The assistant loads INTO the target (ctx_dft == nullptr by design), ++ // so a draft-simple impl would dereference the null draft context and crash. The assistant ++ // path is set up by the has_mtp_assistant branch below. + LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__); + has_draft_simple = true; + } +@@ -1378,6 +1754,20 @@ + if (has_mtp) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); + } ++ if (has_mtp_assistant) { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter (validate in-target assistant) ++ const llama_model * model_tgt = params.draft.ctx_tgt ? llama_get_model(params.draft.ctx_tgt) : nullptr; ++ const llama_model * model_mtp = model_tgt ? llama_model_get_mtp_assistant(model_tgt) : nullptr; ++ if (!model_mtp) { ++ LOG_WRN("%s: 'draft-assistant' requires the gemma4_assistant GGUF loaded into the target " ++ "(--spec-type draft-assistant with --mtp-head/--model-draft); disabling\n", __func__); ++ } else if (!common_speculative_mtp_arch_ok(model_tgt, model_mtp)) { ++ LOG_WRN("%s: 'draft-assistant' requires target arch gemma4 + assistant arch gemma4_assistant; disabling\n", __func__); ++ } else if (!common_speculative_are_compatible_mtp(model_tgt, model_mtp)) { ++ LOG_WRN("%s: 'draft-assistant' assistant failed vocab compatibility; disabling\n", __func__); ++ } else { ++ configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_MTP, params)); ++ } ++ } + } + + std::vector> impls = {}; +@@ -1398,6 +1788,10 @@ + impls.push_back(std::make_unique(config.params, n_seq)); + break; + } ++ case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter ++ impls.push_back(std::make_unique(config.params, n_seq)); ++ break; ++ } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { + common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); + +@@ -1460,6 +1854,12 @@ + return result; + } + ++// opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / ++// set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative ++// framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ ++// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on ++// common_speculative_impl_draft_assistant above). ++ + void common_speculative_free(common_speculative * spec) { + if (spec == nullptr) { + return; +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -540,6 +540,21 @@ + size_t n_paths, + struct llama_model_params params); + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP: load gemma4_assistant GGUF into a gemma4 target (call after ++ // llama_model_load_from_file, before llama_init_from_model). Returns 0 on success. ++ LLAMA_API int llama_model_load_mtp_from_file( ++ struct llama_model * model, ++ const char * path_mtp, ++ struct llama_model_params params); ++ ++ LLAMA_API const struct llama_model * llama_model_get_mtp_assistant(const struct llama_model * model); ++ ++ LLAMA_API bool llama_model_has_mtp_assistant(const struct llama_model * model); ++ ++ // Backbone hidden size for MTP input (0 if no MTP assistant is loaded). ++ LLAMA_API uint32_t llama_model_mtp_n_embd_backbone(const struct llama_model * model); ++ + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); +@@ -1010,6 +1025,22 @@ + struct llama_context * ctx, + struct llama_batch batch); + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, ++ // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). ++ // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) ++ // receive per-step logits and the final backbone hidden. Returns 0 on success. ++ LLAMA_API int32_t llama_decode_mtp( ++ struct llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) +diff --git a/llama.cpp/src/llama-arch.cpp b/llama.cpp/src/llama-arch.cpp +--- a/llama.cpp/src/llama-arch.cpp ++++ b/llama.cpp/src/llama-arch.cpp +@@ -57,6 +57,7 @@ + { LLM_ARCH_GEMMA3, "gemma3" }, + { LLM_ARCH_GEMMA3N, "gemma3n" }, + { LLM_ARCH_GEMMA4, "gemma4" }, ++ { LLM_ARCH_GEMMA4_ASSISTANT, "gemma4-assistant" }, // opencoti F5 M6-S4 mtp — upstream-aligned arch string (#23398, hyphen) + { LLM_ARCH_GEMMA_EMBEDDING, "gemma-embedding" }, + { LLM_ARCH_STARCODER2, "starcoder2" }, + { LLM_ARCH_MAMBA, "mamba" }, +@@ -293,6 +294,14 @@ + { LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" }, + { LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" }, + ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP, opencoti-private additive KV (upstream #23398/#24282 ++ // has no equivalent; backbone width moved to the generic %s.embedding_length_out). %s = "gemma4-assistant". ++ { LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, "%s.n_centroids" }, ++ { LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, "%s.centroid_top_k" }, ++ { LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, "%s.attention.k_eq_v" }, ++ { LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, "%s.use_ordered_embeddings" }, ++ { LLM_KV_GEMMA4_ASSISTANT_REQUIRES_TARGET_ARCH, "%s.requires_target_arch" }, ++ + { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" }, + { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" }, + { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" }, +@@ -452,6 +461,11 @@ + { LLM_TENSOR_NEXTN_HNORM, "blk.%d.nextn.hnorm" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "blk.%d.nextn.shared_head_head" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "blk.%d.nextn.shared_head_norm" }, ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant tensors ++ { LLM_TENSOR_NEXTN_PROJ_PRE, "nextn.pre_projection" }, ++ { LLM_TENSOR_NEXTN_PROJ_POST, "nextn.post_projection" }, ++ { LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" }, ++ { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, + { LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" }, + { LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" }, + { LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" }, +@@ -767,6 +781,11 @@ + {LLM_TENSOR_NEXTN_HNORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant tensors ++ {LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_INPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_MASKED_EMBD_CENTROIDS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_MASKED_EMBD_ORDERING, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, + // Nemotron 3 Super + // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU + {LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, +diff --git a/llama.cpp/src/llama-arch.h b/llama.cpp/src/llama-arch.h +--- a/llama.cpp/src/llama-arch.h ++++ b/llama.cpp/src/llama-arch.h +@@ -61,6 +61,7 @@ + LLM_ARCH_GEMMA3, + LLM_ARCH_GEMMA3N, + LLM_ARCH_GEMMA4, ++ LLM_ARCH_GEMMA4_ASSISTANT, // opencoti F5 M6-S4 mtp + LLM_ARCH_GEMMA_EMBEDDING, + LLM_ARCH_STARCODER2, + LLM_ARCH_MAMBA, +@@ -345,6 +346,16 @@ + LLM_KV_DENSE_2_FEAT_OUT, + LLM_KV_DENSE_3_FEAT_IN, + LLM_KV_DENSE_3_FEAT_OUT, ++ ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP, opencoti-private additive KV (no upstream KV ++ // equivalent; upstream #24282 detects the ordered/masked-embedder head by tensor presence, not KV ++ // flags). Backbone width is NOT here: it reuses the generic LLM_KV_EMBEDDING_LENGTH_OUT, matching ++ // upstream. The "%s." prefix auto-follows the arch string (now "gemma4-assistant"). ++ LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, ++ LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, ++ LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, ++ LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, ++ LLM_KV_GEMMA4_ASSISTANT_REQUIRES_TARGET_ARCH, + }; + + enum llm_tensor { +@@ -556,6 +567,15 @@ + LLM_TENSOR_NEXTN_HNORM, + LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, + LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, ++ ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 ++ // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile ++ // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our ++ // single-context build_attn_mtp cross-read — NOT upstream's ctx_other. See plan. ++ LLM_TENSOR_NEXTN_PROJ_PRE, ++ LLM_TENSOR_NEXTN_PROJ_POST, ++ LLM_TENSOR_MASKED_EMBD_CENTROIDS, ++ LLM_TENSOR_MASKED_EMBD_ORDERING, + }; + + enum llm_tensor_layer { +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -9,6 +9,7 @@ + #include "llama-memory.h" + #include "llama-mmap.h" + #include "llama-model.h" ++#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp + #include "llama-ext.h" + #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" +@@ -432,6 +433,17 @@ + } + + llama_context::~llama_context() { ++ // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down ++ // sched_mtp and the backends it touches. ++ if (mtp_worker.joinable()) { ++ { ++ std::lock_guard lk(mtp_mu); ++ mtp_worker_stop.store(true, std::memory_order_release); ++ } ++ mtp_cv_request.notify_all(); ++ mtp_worker.join(); ++ } ++ + if (!model.hparams.no_alloc) { + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; +@@ -1313,6 +1325,208 @@ + return res; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++bool llama_context::ensure_sched_mtp() { ++ // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count ++ // (mtp_fused_steps). A fused N-step graph has ~N× the nodes of the single-step graph and a ++ // distinct topology, so when a larger block is requested we drop and rebuild the reserve. ++ const int32_t need = std::max(mtp_fused_steps, 1); ++ if (sched_mtp && mtp_reserved_steps >= need) { ++ return true; ++ } ++ if (!model.mtp_assistant) { ++ return false; ++ } ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ ++ const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); ++ const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; ++ ++ gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); ++ sched_mtp.reset(ggml_backend_sched_new( ++ backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), ++ max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); ++ if (!sched_mtp) { ++ LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. ++ // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), ++ // so a single reserve covers all subsequent decode_mtp calls. ++ { ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ llama_memory_context_ptr mctx = memory->init_full(); ++ if (!mctx) { ++ LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(0); ++ data->seq_idx[0] = 0; ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = 0; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ const uint32_t save_n_outputs = n_outputs; ++ n_outputs = 1; ++ ++ auto * res = gf_res_prev_mtp.get(); ++ const auto gparams = graph_params_mtp(res, ub, mctx.get()); ++ res->reset(); ++ ggml_backend_sched_reset(sched_mtp.get()); ++ ++ auto * gf = model.build_graph(gparams); ++ n_outputs = save_n_outputs; ++ ++ if (!gf) { ++ LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { ++ LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ // Discard the reserve graph cache so the first real call rebuilds with proper inputs. ++ res->reset(); ++ ggml_backend_sched_reset(sched_mtp.get()); ++ } ++ ++ // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in ++ // decode_mtp_async (only when the opt-in async path is actually taken), NOT here — so the ++ // default in-thread sync path creates no extra thread and is runtime-identical to P3. ++ ++ mtp_reserved_steps = need; // reserve now covers `need` fused steps ++ return true; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++llm_graph_result * llama_context::process_ubatch_mtp( ++ const llama_ubatch & ubatch, ++ llama_memory_context_i * mctx, ++ ggml_status & ret) { ++ GGML_ASSERT(sched_mtp && gf_res_prev_mtp); ++ ++ auto * res = gf_res_prev_mtp.get(); ++ auto * gf = res->get_gf(); ++ ++ // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it ++ // explicitly here in case a concurrent target decode mutates it. The MTP graph ++ // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. ++ llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); ++ gparams.n_outputs = 1; ++ gparams.sched = sched_mtp.get(); ++ ++ if (!graph_reuse_disable && res->can_reuse(gparams)) { ++ // sched_mtp does not use pipeline parallelism (created with pipeline=false), ++ // so no synchronize is needed before set_inputs. ++ } else { ++ res->reset(); ++ ++ ggml_backend_sched_reset(sched_mtp.get()); ++ ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); ++ ++ gf = model.build_graph(gparams); ++ if (!gf) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); ++ ret = GGML_STATUS_FAILED; ++ return nullptr; ++ } ++ if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { ++ LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); ++ ret = GGML_STATUS_ALLOC_FAILED; ++ return nullptr; ++ } ++ } ++ ++ res->set_inputs(&ubatch); ++ ++ const auto status = graph_compute_mtp(res->get_gf()); ++ ++ if (status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); ++ ret = status; ++ return nullptr; ++ } ++ ++ ret = GGML_STATUS_SUCCESS; ++ return res; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { ++ // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s ++ // threadpool/n_threads dance only for the CPU backend; the MTP head is small and ++ // thread saturation matters less, but consistency with graph_compute() avoids ++ // surprising backend reconfigurations between target and MTP runs. ++ // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker ++ // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. ++ const int n_threads = cparams.n_threads; ++ ggml_threadpool_t tp = threadpool; ++ ++ if (backend_cpu != nullptr) { ++ auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); ++ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ++ ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); ++ if (set_threadpool_fn) { ++ set_threadpool_fn(backend_cpu, tp); ++ } ++ } ++ for (const auto & set_n_threads_fn : set_n_threads_fns) { ++ set_n_threads_fn.second(set_n_threads_fn.first, n_threads); ++ } ++ ++ auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); ++ if (status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", ++ __func__, status); ++ } ++ return status; ++} ++ + llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { + if (mctx && !mctx->apply()) { + LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); +@@ -1773,6 +1987,14 @@ + + bool did_optimize = false; + ++ // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its ++ // KV below (memory_update's pending shifts/copies, then init_batch's slot apply — the S1 ++ // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this ++ // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in ++ // the current sync-facade integration (nothing is in flight here) and a single pointer ++ // check when no MTP assistant is loaded. ++ mtp_drain_before_mutate(); ++ + // handle any pending shifts/copies + memory_update(false); + +@@ -2347,6 +2569,571 @@ + return gf; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++llm_graph_params llama_context::graph_params_mtp( ++ llm_graph_result * res, ++ const llama_ubatch & ubatch, ++ const llama_memory_context_i * mctx) const { ++ const llama_model * mtp = model.mtp_assistant.get(); ++ GGML_ASSERT(mtp); ++ ++ return { ++ /*.arch =*/ mtp->arch, ++ /*.hparams =*/ mtp->hparams, ++ /*.cparams =*/ cparams, ++ /*.ubatch =*/ ubatch, ++ /*.gtype =*/ LLM_GRAPH_TYPE_MTP, ++ /*.sched =*/ sched.get(), ++ /*.backend_cpu =*/ backend_cpu, ++ /*.cvec =*/ cvec.get(), ++ /*.loras =*/ loras.get(), ++ /*.mctx =*/ mctx, ++ /*.cross =*/ &cross, ++ /*.samplers =*/ sampling.samplers, ++ /*.n_outputs =*/ n_outputs, ++ /*.cb =*/ graph_get_cb(), ++ /*.res =*/ res, ++ /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) ++ }; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. ++int32_t llama_context::decode_mtp( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams ++ // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the ++ // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes async→wait. ++ // With no target/draft overlap this is byte-identical to the P3 sync path — the worker runs ++ // the same N-step loop off-thread while we block in decode_mtp_wait. ++ if (out_logits) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_logits, out_h_prev_last); ++ } ++ // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the ++ // fused single-graph draft (decode_mtp_fused) — ONE launch+sync+D2H for the whole block, the ++ // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused ++ // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse → sequential. ++ static const bool fused_enabled = [] { ++ const char * e = getenv("OPENCOTI_MTP_FUSED"); ++ return !(e && e[0] == '0'); ++ }(); ++ if (fused_enabled && n_steps > 1) { ++ return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_h_prev_last); ++ } ++ // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). ++ // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker ++ // gives NO throughput benefit — real depth-2 overlap is deferred (limited by the h-prev ++ // dependency; even the AtomicBot fork ships depth-1) — AND the worker-thread CUDA execution ++ // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off ++ // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. ++ static const bool async_enabled = [] { ++ const char * e = getenv("OPENCOTI_MTP_ASYNC"); ++ return e && e[0] == '1'; ++ }(); ++ if (!async_enabled) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_logits, out_h_prev_last); ++ } ++ const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); ++ if (rc != 0) { ++ return rc; ++ } ++ return decode_mtp_wait(out_drafts, out_h_prev_last); ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++int32_t llama_context::decode_mtp_sync( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, ++ // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) ++ mtp_fused_steps = 1; ++ ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ ++ const int32_t n_vocab = model.vocab.n_tokens(); ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); ++ return -4; ++ } ++ ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(seq_id); ++ data->seq_idx[(size_t) seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post ++ // back). It MUST be flagged output=1 so build_inp_out_ids selects its row — and so the live ++ // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here ++ // produced a divergent graph whose output-selection tensor was left with a null backend ++ // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. ++ data->output[0] = 1; ++ ++ if (n_steps <= 0) { ++ return 0; ++ } ++ ++ // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token ++ // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. ++ for (int32_t k = 0; k < n_steps; ++k) { ++ data->token[0] = last_token; ++ data->pos[0] = attn_pos + 1 + (llama_pos) k; ++ std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); ++ ++ int32_t best_i32 = 0; ++ ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); ++ out_drafts[k] = (llama_token) best_i32; ++ ++ if (out_logits) { ++ ggml_tensor * t_logits = res->get_logits(); ++ GGML_ASSERT(t_logits); ++ ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, ++ 0, (size_t) n_vocab * sizeof(float)); ++ } ++ ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); ++ ++ last_token = (llama_token) best_i32; ++ } ++ ++ if (out_h_prev_last) { ++ std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); ++ } ++ ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head ++// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls ++// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index — already ++// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. ++// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, ++// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). ++// ++// One mask shared across steps (built from step 0's position). For FULL-attn target layers and ++// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa — the short-context / ++// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are ++// byte-identical to the sequential path. For SWA layers at long context the true per-step window ++// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is ++// output-safe — the target verify pass rejects any mismatched draft, so the emitted stream is ++// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix ++// is a per-step mask slice — deferred until measured. See buglog bug-420.) ++int32_t llama_context::decode_mtp_fused( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ if (n_steps <= 0) { ++ return 0; ++ } ++ // A single step has nothing to fuse — defer to the proven sequential path. ++ if (n_steps == 1) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, /*out_logits=*/nullptr, out_h_prev_last); ++ } ++ ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); ++ return -4; ++ } ++ ++ // Build/reserve a fused n_steps graph. ++ mtp_fused_steps = n_steps; ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ ++ // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step ++ // query positions consumed by the wrapper's inp_pos_steps (set_input). ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(n_steps); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(seq_id); ++ data->seq_idx[(size_t) seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = (uint32_t) n_steps; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; ++ ++ data->token[0] = last_token; ++ std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ data->pos[k] = attn_pos + 1 + (llama_pos) k; ++ } ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); ++ GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); ++ ++ std::vector drafts((size_t) n_steps); ++ ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ out_drafts[k] = (llama_token) drafts[(size_t) k]; ++ } ++ ++ if (out_h_prev_last) { ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); ++ } ++ ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one ++// in-flight request per context (returns -7 if a prior request was not yet waited). ++int32_t llama_context::decode_mtp_async( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0 || !h_prev || n_steps <= 0) { ++ LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", ++ __func__, n_bb, (const void *) h_prev, n_steps); ++ return -4; ++ } ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ if (!mtp_worker.joinable()) { ++ mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); ++ } ++ ++ { ++ std::unique_lock lk(mtp_mu); ++ if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { ++ LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", ++ __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, ++ (int) mtp_completed.has_value()); ++ return -7; ++ } ++ mtp_request req; ++ req.seq_id = seq_id; ++ req.attn_pos = attn_pos; ++ req.last_token = last_token; ++ req.n_steps = n_steps; ++ req.h_prev.assign(h_prev, h_prev + n_bb); ++ mtp_pending = std::move(req); ++ } ++ mtp_cv_request.notify_one(); ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last ++// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). ++int32_t llama_context::decode_mtp_wait( ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ std::unique_lock lk(mtp_mu); ++ mtp_cv_response.wait(lk, [this] { ++ return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); ++ }); ++ if (!mtp_completed.has_value()) { ++ LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); ++ return -7; ++ } ++ mtp_response resp = std::move(*mtp_completed); ++ mtp_completed.reset(); ++ lk.unlock(); ++ ++ if (resp.status != 0) { ++ return resp.status; ++ } ++ if (out_drafts && !resp.drafts.empty()) { ++ std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); ++ } ++ if (out_h_prev_last && !resp.h_prev_last.empty()) { ++ std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); ++ } ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's ++// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the ++// single ubatch row MUST be flagged output — a 0 here reintroduces the null-out-ids abort ++// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. ++int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(req.seq_id); ++ data->seq_idx[(size_t) req.seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = req.seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 ++ ++ std::vector h(req.h_prev); ++ llama_token last_token = req.last_token; ++ resp.drafts.assign((size_t) req.n_steps, 0); ++ ++ for (int32_t k = 0; k < req.n_steps; ++k) { ++ data->token[0] = last_token; ++ data->pos[0] = req.attn_pos + 1 + (llama_pos) k; ++ std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); ++ int32_t best_i32 = 0; ++ ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); ++ last_token = (llama_token) best_i32; ++ resp.drafts[(size_t) k] = last_token; ++ ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); ++ } ++ ++ resp.h_prev_last = std::move(h); ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on ++// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. ++void llama_context::mtp_worker_loop() { ++ for (;;) { ++ mtp_request req; ++ { ++ std::unique_lock lk(mtp_mu); ++ mtp_cv_request.wait(lk, [this] { ++ return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); ++ }); ++ if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { ++ return; ++ } ++ req = std::move(*mtp_pending); ++ mtp_pending.reset(); ++ mtp_in_flight = true; ++ } ++ ++ mtp_response resp; ++ resp.status = decode_mtp_run(req, resp); ++ ++ { ++ std::lock_guard lk(mtp_mu); ++ mtp_in_flight = false; ++ mtp_completed = std::move(resp); ++ } ++ mtp_cv_response.notify_one(); ++ } ++} ++ ++// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV ++// read retired), then sync sched_mtp — WITHOUT consuming mtp_completed (the spec loop's _wait ++// still needs the drafts). Called at decode() top before any KV mutation. Inert when no ++// assistant is loaded or nothing is in flight (drained==false ⇒ no extra sched sync). ++void llama_context::mtp_drain_before_mutate() { ++ if (!model.mtp_assistant) { ++ return; ++ } ++ bool drained = false; ++ { ++ std::unique_lock lk(mtp_mu); ++ if (mtp_pending.has_value() || mtp_in_flight) { ++ mtp_cv_response.wait(lk, [this] { ++ return !mtp_in_flight && !mtp_pending.has_value(); ++ }); ++ drained = true; ++ } ++ } ++ if (drained && sched_mtp) { ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ } ++} ++ + llm_graph_params llama_context::graph_params( + llm_graph_result * res, + const llama_ubatch & ubatch, +@@ -4043,6 +4830,24 @@ + return ret; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++int32_t llama_decode_mtp( ++ llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ if (!ctx) { ++ LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); ++ return -1; ++ } ++ return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); ++} ++ + // + // perf + // +diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h +--- a/llama.cpp/src/llama-context.h ++++ b/llama.cpp/src/llama-context.h +@@ -13,6 +13,13 @@ + #include + #include + ++// opencoti F5 M6-S4 mtp (P4): async MTP draft worker thread + single-slot request/response queue. ++#include ++#include ++#include ++#include ++#include ++ + struct llama_model; + class llama_batch_allocr; + +@@ -135,6 +142,65 @@ + llama_memory_context_i * mctx, + ggml_status & ret); + ++ // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. ++ llm_graph_params graph_params_mtp( ++ llm_graph_result * res, ++ const llama_ubatch & ubatch, ++ const llama_memory_context_i * mctx) const; ++ ++ int32_t decode_mtp( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ ++ int32_t decode_mtp_sync( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps ++ // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft ++ // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced ++ // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits ++ // verification path stays on decode_mtp_sync. ++ int32_t decode_mtp_fused( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ ++ // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync ++ // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams ++ // no per-step logits); otherwise it submits via decode_mtp_async and blocks in ++ // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the ++ // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). ++ int32_t decode_mtp_async( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps); ++ int32_t decode_mtp_wait( ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before ++ // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP ++ // assistant is loaded or nothing is in flight (the common case). ++ void mtp_drain_before_mutate(); ++ + int encode(const llama_batch & batch_inp); + int decode(const llama_batch & batch_inp); + +@@ -251,6 +317,14 @@ + const llama_memory_context_i * mctx, + llm_graph_type gtype) const; + ++ // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. ++ bool ensure_sched_mtp(); ++ llm_graph_result * process_ubatch_mtp( ++ const llama_ubatch & ubatch, ++ llama_memory_context_i * mctx, ++ ggml_status & ret); ++ ggml_status graph_compute_mtp(ggml_cgraph * gf); ++ + llm_graph_cb graph_get_cb() const; + + // TODO: read/write lora adapters and cvec +@@ -350,6 +424,46 @@ + llm_graph_result_ptr gf_res_prev; + llm_graph_result_ptr gf_res_reserve; + ++ // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be ++ // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable ++ // single-token MTP graph result across steps of one decode_mtp call. ++ ggml_backend_sched_ptr sched_mtp; ++ llm_graph_result_ptr gf_res_prev_mtp; ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for ++ // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it ++ // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so ++ // ensure_sched_mtp only re-reserves when a larger fused block is requested. ++ int32_t mtp_fused_steps = 1; ++ int32_t mtp_reserved_steps = 0; ++ ++ // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. ++ // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp ++ // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync ++ // facade), so the worker never races the target's sched. The destructor joins the worker. ++ struct mtp_request { ++ llama_seq_id seq_id = 0; ++ llama_pos attn_pos = 0; ++ llama_token last_token = 0; ++ std::vector h_prev; ++ int32_t n_steps = 0; ++ }; ++ struct mtp_response { ++ int32_t status = 0; ++ std::vector drafts; ++ std::vector h_prev_last; ++ }; ++ std::thread mtp_worker; ++ std::atomic mtp_worker_stop{false}; ++ std::mutex mtp_mu; ++ std::condition_variable mtp_cv_request; ++ std::condition_variable mtp_cv_response; ++ std::optional mtp_pending; // submitted, not yet picked up ++ bool mtp_in_flight = false; // worker is processing ++ std::optional mtp_completed; // worker finished, awaiting _wait ++ int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop ++ void mtp_worker_loop(); ++ + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_ptr buf_output; + +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -106,6 +106,38 @@ + return res; + } + ++// opencoti F5 M6-S4 mtp ++void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { ++ GGML_ASSERT(ubatch && ubatch->n_tokens == 1); ++ GGML_ASSERT(ubatch->token && ubatch->embd); ++ GGML_ASSERT(inp_last_token && inp_h_prev); ++ ++ ggml_backend_tensor_set(inp_last_token, ubatch->token, 0, sizeof(llama_token)); ++ ++ const int64_t n_bb = inp_h_prev->ne[0]; ++ ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] ++ // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. ++ for (size_t k = 0; k < inp_pos_steps.size(); ++k) { ++ ggml_backend_tensor_set(inp_pos_steps[k], ubatch->pos + k, 0, sizeof(int32_t)); ++ } ++} ++ ++// opencoti F5 M6-S4 mtp ++bool llm_graph_input_mtp::can_reuse(const llm_graph_params & params) { ++ if (params.gtype != LLM_GRAPH_TYPE_MTP) { ++ return false; ++ } ++ const bool base = inp_last_token && inp_last_token->ne[0] == 1 && inp_h_prev && inp_h_prev->ne[1] == 1; ++ // opencoti F5 M6-S4 mtp (P4 fused): a fused N-step graph and a 1-step graph have different ++ // topology — only reuse when the stored per-step position count matches the requested steps. ++ if (params.n_mtp_steps > 1) { ++ return base && inp_pos_steps.size() == (size_t) params.n_mtp_steps; ++ } ++ return base && inp_pos_steps.empty(); ++} ++ + void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { + // opencoti F5 dca #623: in a DCA-on graph the full-attention layers rope Q from inp_dca's own + // per-regime positions (build_attn_dca), so the standard inp_pos is consumed by no node on archs +@@ -516,35 +548,48 @@ + } + + void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { +- // base tensors may not be allocated if there are no non-SWA attention layers ++ // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently — ++ // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP ++ // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but ++ // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) ++ // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the ++ // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP ++ // case (idxs pruned, mask live) → stale mask, silent divergence (bug-404). Decoupling restores ++ // "set each idxs/mask/rot iff IT is allocated"; for every non-MTP graph idxs+mask share ++ // allocation fate, so this is byte-identical to stock. Matches the AtomicBot fork's guarded iswa. ++ ++ // base index writes (k+v share allocation fate): present iff there are non-SWA write layers. + if (self_k_idxs && self_k_idxs->buffer) { + mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch); + mctx->get_base()->set_input_v_idxs(self_v_idxs, ubatch); +- ++ } ++ // base mask is consumed by the MTP cross-read even when the base idxs are pruned — guard alone. ++ if (self_kq_mask && self_kq_mask->buffer) { + mctx->get_base()->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + } + +- // swa tensors may not be allocated if there are no SWA attention layers ++ // swa index writes (k+v share allocation fate): present iff there are SWA write layers. + if (self_k_idxs_swa && self_k_idxs_swa->buffer) { + mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch); + mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch); +- ++ } ++ if (self_kq_mask_swa && self_kq_mask_swa->buffer) { + mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); + } + +- if (self_k_rot) { ++ if (self_k_rot && self_k_rot->buffer) { + mctx->get_base()->set_input_k_rot(self_k_rot); + } + +- if (self_v_rot) { ++ if (self_v_rot && self_v_rot->buffer) { + mctx->get_base()->set_input_v_rot(self_v_rot); + } + +- if (self_k_rot_swa) { ++ if (self_k_rot_swa && self_k_rot_swa->buffer) { + mctx->get_swa()->set_input_k_rot(self_k_rot_swa); + } + +- if (self_v_rot_swa) { ++ if (self_v_rot_swa && self_v_rot_swa->buffer) { + mctx->get_swa()->set_input_v_rot(self_v_rot_swa); + } + } +@@ -844,6 +889,7 @@ + t_logits = nullptr; + t_embd = nullptr; + t_embd_pooled = nullptr; ++ t_argmax = nullptr; // opencoti F5 M6-S4 mtp + t_sampled.clear(); + t_sampled_probs.clear(); + t_sampled_logits.clear(); +@@ -2747,6 +2793,91 @@ + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } ++ ++ return cur; ++} ++ ++// opencoti F5 M6-S4 mtp ++ggml_tensor * llm_graph_context::build_attn_mtp( ++ llm_graph_input_attn_kv_iswa * inp, ++ ggml_tensor * wo, ++ ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ++ ggml_tensor * kq_b, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il_mtp, ++ int32_t il_kv_tgt, ++ bool read_from_swa_kv, ++ int64_t kv_embd_head_v, ++ int64_t kv_n_head_v, ++ bool use_k_as_v) const { ++ auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; ++ auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; ++ ++ if (k_rot) { ++ q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); ++ } ++ if (v_rot) { ++ // V-only rotation is applied after MHA in the standard path; no V write here. ++ } ++ ++ ggml_build_forward_expand(gf, q_cur); ++ ++ const auto * mctx_iswa = inp->mctx; ++ const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); ++ ++ const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); ++ ++ ggml_tensor * q = q_cur; ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); ++ ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); ++ ++ if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) { ++ // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the ++ // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp ++ // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at ++ // head_dim∈{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 ++ // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired ++ // inverse on the output → wrong logits (graph still *built*, which is why P2's inert build ++ // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity ++ // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. ++ if (q->ne[0] % 128 != 0) { ++ const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; ++ q = ggml_pad(ctx0, q, pad, 0, 0, 0); ++ } ++ if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } ++ } ++ ++ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); ++ cb(cur, "kqv_out_mtp", il_mtp); ++ ++ if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0) { ++ const int64_t orig_v_head = kv_embd_head_v; ++ const int64_t padded_v_head = v->ne[0]; ++ if (padded_v_head != orig_v_head) { ++ const int64_t n_head_v = kv_n_head_v; ++ const int64_t n_tokens_cur = cur->ne[1]; ++ cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); ++ cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, ++ cur->nb[1], cur->nb[2], 0); ++ cur = ggml_cont(ctx0, cur); ++ cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); ++ } ++ } ++ ++ if (v_rot) { ++ cur = ggml_mul_mat_aux(ctx0, cur, v_rot); ++ } ++ ++ if (wo) { ++ cur = build_lora_mm(wo, cur); ++ cb(cur, "mtp_wo_out", il_mtp); ++ } ++ if (wo_b) { ++ cur = ggml_add(ctx0, cur, wo_b); ++ } + + return cur; + } +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -36,6 +36,7 @@ + LLM_GRAPH_TYPE_ENCODER, + LLM_GRAPH_TYPE_DECODER, + LLM_GRAPH_TYPE_DECODER_MTP, ++ LLM_GRAPH_TYPE_MTP, // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter graph (distinct from native DECODER_MTP/NextN) + }; + + enum llm_ffn_op_type { +@@ -124,6 +125,25 @@ + const int64_t n_embd = 0; + }; + ++// opencoti F5 M6-S4 mtp ++// Gemma 4 MTP: last target token id + backbone hidden (n_bb floats) for a single step. ++class llm_graph_input_mtp : public llm_graph_input_i { ++public: ++ llm_graph_input_mtp() = default; ++ ~llm_graph_input_mtp() override = default; ++ ++ void set_input(const llama_ubatch * ubatch) override; ++ bool can_reuse(const llm_graph_params & params) override; ++ ++ ggml_tensor * inp_last_token = nullptr; // I32 [1] ++ ggml_tensor * inp_h_prev = nullptr; // F32 [n_bb, 1] ++ // opencoti F5 M6-S4 mtp (P4 fused): one I32[1] RoPE position per fused draft step. ++ // Empty for the single-step path (which uses build_inp_pos). For n_mtp_steps>1 the draft ++ // chain is unrolled in-graph: each step's token/h chain from the previous step's argmax/ ++ // backbone, but the per-step query position still comes from the host via these tensors. ++ std::vector inp_pos_steps; ++}; ++ + class llm_graph_input_pos : public llm_graph_input_i { + public: + llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} +@@ -572,9 +592,20 @@ + + llm_graph_result * res; + ++ // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. ++ // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone ++ // into the next step with no host round-trip / per-step GPU sync (the throughput fix). ++ // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. ++ int32_t n_mtp_steps = 1; ++ + // return true if the "other" params would result in a graph with the same topology as with the current params + // having the same topology allows us to reuse the graph in some cases + bool allow_reuse(const llm_graph_params & other) const { ++ // opencoti F5 M6-S4 mtp: 0.10.3's native allow_reuse condition below already permits reuse when ++ // BOTH token+embd are present in both ubatches (the third OR-clause) — exactly the Gemma-4 MTP ++ // case — so the 0.10.1-era token_compat relaxation is redundant and intentionally not restored. ++ // The fused n_mtp_steps>1 graph reuse (the P4 throughput win) is preserved by that native clause. ++ + // first check the ubatch + bool can_reuse_ubatch = + ubatch.equal_seqs() == other.ubatch.equal_seqs() && +@@ -651,6 +682,12 @@ + ggml_tensor * get_embd_pooled() const { return t_embd_pooled; } + ggml_tensor * get_h_pre_norm() const { return t_h_pre_norm; } + ++ // opencoti F5 M6-S4 mtp ++ // Optional in-graph argmax tensor (I32 [n_outputs]). Currently set only by the ++ // Gemma4 MTP graph so the host can read a 4-byte argmax instead of an ++ // n_vocab-float logits row + CPU argmax loop. ++ ggml_tensor * get_argmax() const { return t_argmax; } ++ + ggml_cgraph * get_gf() const { return gf; } + ggml_context * get_ctx() const { return ctx_compute.get(); } + +@@ -679,6 +716,7 @@ + ggml_tensor * t_embd = nullptr; + ggml_tensor * t_embd_pooled = nullptr; + ggml_tensor * t_h_pre_norm = nullptr; // [n_embd, n_outputs] hidden state before final output norm ++ ggml_tensor * t_argmax = nullptr; // opencoti F5 M6-S4 mtp: optional, currently MTP-only (see get_argmax) + + std::map t_sampled_logits; + std::map t_candidates; +@@ -983,6 +1021,26 @@ + int il, + bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). ++ // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). ++ // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). ++ ggml_tensor * build_attn_mtp( ++ llm_graph_input_attn_kv_iswa * inp, ++ ggml_tensor * wo, ++ ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ++ ggml_tensor * kq_b, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il_mtp, ++ int32_t il_kv_tgt, ++ bool read_from_swa_kv, ++ int64_t kv_embd_head_v, ++ int64_t kv_n_head_v, ++ bool use_k_as_v) const; ++ + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( +diff --git a/llama.cpp/src/llama-hparams.h b/llama.cpp/src/llama-hparams.h +--- a/llama.cpp/src/llama-hparams.h ++++ b/llama.cpp/src/llama-hparams.h +@@ -217,6 +217,14 @@ + // gemma4 per-layer embedding + uint32_t n_embd_per_layer = 0; + ++ // opencoti F5 M6-S4 mtp — gemma4 assistant-MTP (speculative drafter), opencoti-private fields. ++ // Backbone width is NOT a private field: it reuses n_embd_out_impl (above), loaded from the generic ++ // upstream KV %s.embedding_length_out (#23398) — so this struct stays free of a redundant duplicate. ++ uint32_t n_centroids = 0; ++ uint32_t centroid_top_k = 0; ++ bool attention_k_eq_v = false; ++ bool use_ordered_embeddings = false; ++ + // needed by encoder-decoder models (e.g. T5, FLAN-T5) + // ref: https://github.com/ggml-org/llama.cpp/pull/8141 + llama_token dec_start_token_id = LLAMA_TOKEN_NULL; +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -229,6 +229,21 @@ + return std::make_unique(this); + } + ++// opencoti F5 M6-S4 mtp ++llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { ++ llama_kv_cache::slot_info_vec_t sinfos_base; ++ llama_kv_cache::slot_info_vec_t sinfos_swa; ++ ++ sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); ++ sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); ++ ++ std::vector ubatches; ++ ubatches.push_back(std::move(ubatch)); ++ ++ return std::make_unique( ++ this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); ++} ++ + llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); + } +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -60,6 +60,9 @@ + + llama_memory_context_ptr init_full() override; + ++ // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). ++ llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); ++ + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -1827,6 +1827,39 @@ + return updated; + } + ++// opencoti F5 M6-S4 mtp ++llama_kv_cache::slot_info llama_kv_cache::mtp_slot_info(llama_seq_id seq_id) const { ++ GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); ++ ++ const uint32_t st = seq_to_stream[seq_id]; ++ const auto & cells = v_cells[st]; ++ ++ llama_pos pmax = cells.seq_pos_max(seq_id); ++ ++ uint32_t idx = 0; ++ ++ if (pmax >= 0) { ++ for (uint32_t i = 0; i < cells.size(); ++i) { ++ if (!cells.seq_has(i, seq_id)) { ++ continue; ++ } ++ if (cells.pos_get(i) == pmax) { ++ idx = i; ++ break; ++ } ++ } ++ } ++ ++ slot_info res; ++ res.s0 = 0; ++ res.s1 = 0; ++ res.strm = { (llama_seq_id) st }; ++ res.idxs.resize(1); ++ res.idxs[0] = { idx }; ++ ++ return res; ++} ++ + llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, bool cont) const { + + if (debug > 0) { +@@ -5039,6 +5072,16 @@ + llama_kv_cache * kv, + llama_kv_cache::slot_info_vec_t sinfos, + std::vector ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { ++ // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, ++ // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from ++ // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a ++ // view with a garbage dim-2 (observed ne[2]=2956782432 → ggml.c view-bounds assert). For ++ // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. ++ if (!this->sinfos.empty()) { ++ n_kv = (int32_t) kv->get_n_kv(this->sinfos[0]); ++ } else { ++ n_kv = 0; ++ } + } + + llama_kv_cache_context::~llama_kv_cache_context() = default; +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -288,6 +288,10 @@ + // return empty vector on failure + slot_info_vec_t prepare(const std::vector & ubatches); + ++ // opencoti F5 M6-S4 mtp: read-only slot_info addressing the last resident cell of seq_id. ++ // Used by the MTP cross-read path (no KV write); points the MTP graph at the target cache. ++ slot_info mtp_slot_info(llama_seq_id seq_id) const; ++ + bool update(llama_context * lctx, bool do_shift, const stream_copy_info & sc_info); + + // find a slot of kv cells that can hold the ubatch +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -136,6 +136,8 @@ + return new llama_model_gemma3n(params); + case LLM_ARCH_GEMMA4: + return new llama_model_gemma4(params); ++ case LLM_ARCH_GEMMA4_ASSISTANT: // opencoti F5 M6-S4 mtp: Gemma-4 MTP assistant (loaded into target) ++ return new llama_model_gemma4_assistant(params); + case LLM_ARCH_GEMMA_EMBEDDING: + return new llama_model_gemma_embedding(params); + case LLM_ARCH_STARCODER2: +@@ -1577,6 +1579,19 @@ + tn, ne, flags); + } + ++// opencoti F5 M6-S4 mtp (#408/bug-408): force OUTPUT-buft placement for a tensor that classifies as ++// LLM_TENSOR_LAYER_INPUT. ml.create_tensor() selects the buft_list by the tensor's llm_tensor_info layer; ++// TOKEN_EMBD is LAYER_INPUT, so passing dev_output.buft_list in the INPUT slot routes it onto the GPU ++// output buft. Used for the gemma4-assistant TIED OUTPUT head so the per-step full-vocab build_lora_mm ++// matmul stays on GPU (the P4/#408 win the 0.10.3 bump dropped — see src/models/gemma4-assistant.cpp). ++// NOT TENSOR_DUPLICATED: the assistant's token_embd is the SOLE use of that GGUF tensor (output head ++// only; input embeddings come from the TARGET), so it must use normal create/load accounting. ++ggml_tensor * llama_model_base::create_tensor_output_head(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { ++ return ml.create_tensor( ++ hparams, &pimpl->cpu_buft_list, pimpl->dev_output.buft_list, pimpl->dev_output.buft_list, nullptr, ++ tn, ne, flags); ++} ++ + std::string llama_model::arch_name() const { + return llm_arch_name(arch); + } +@@ -2349,6 +2364,7 @@ + case LLM_ARCH_GEMMA3: + case LLM_ARCH_GEMMA3N: + case LLM_ARCH_GEMMA4: ++ case LLM_ARCH_GEMMA4_ASSISTANT: // opencoti F5 M6-S4 mtp + case LLM_ARCH_GEMMA_EMBEDDING: + case LLM_ARCH_STARCODER2: + case LLM_ARCH_OPENELM: +@@ -2564,6 +2580,12 @@ + return create_tensor(*ml, tn, ne, flags); + } + ++// opencoti F5 M6-S4 mtp (#408/bug-408): ml-less convenience overload, mirroring create_tensor above. ++ggml_tensor * llama_model_base::create_tensor_output_head(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { ++ GGML_ASSERT(ml != nullptr); ++ return create_tensor_output_head(*ml, tn, ne, flags); ++} ++ + void llama_model_base::create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) { + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED); + if (layer.ffn_gate_up_exps == nullptr) { +diff --git a/llama.cpp/src/llama-model.h b/llama.cpp/src/llama-model.h +--- a/llama.cpp/src/llama-model.h ++++ b/llama.cpp/src/llama-model.h +@@ -559,6 +559,17 @@ + struct ggml_tensor * per_layer_model_proj = nullptr; + struct ggml_tensor * per_layer_proj_norm = nullptr; + ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant weights ++ // (populated only when arch == GEMMA4_ASSISTANT nested model) ++ struct ggml_tensor * mtp_pre_projection = nullptr; ++ struct ggml_tensor * mtp_post_projection = nullptr; ++ struct ggml_tensor * mtp_centroids = nullptr; ++ struct ggml_tensor * mtp_token_ordering = nullptr; ++ ++ // opencoti F5 M6-S4 mtp — nested assistant model loaded via ++ // llama_model_load_mtp_from_file (owns the mtp_* tensors above) ++ std::unique_ptr mtp_assistant; ++ + std::vector layers; + + //Dense linear projections for SentenceTransformers models like embeddinggemma +@@ -669,6 +680,14 @@ + // convenience overload of create_tensor that doesn't require llama_model_loader + ggml_tensor * create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); + ++ // opencoti F5 M6-S4 mtp (#408/bug-408): create a tensor on the OUTPUT device buft regardless of its ++ // classification. For the gemma4-assistant TIED OUTPUT head (tok_embd, used ONLY as the per-step ++ // full-vocab build_lora_mm weight — never an input get_rows) this keeps the matmul on the GPU instead ++ // of a graph-fragmenting CPU island. Forwards dev_output.buft_list in the input slot (TOKEN_EMBD ++ // classifies LAYER_INPUT) with NORMAL load accounting (no TENSOR_DUPLICATED — sole use of the tensor). ++ ggml_tensor * create_tensor_output_head(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); ++ ggml_tensor * create_tensor_output_head(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); ++ + // helper: try merged gate_up_exps first, fall back to separate gate and up + void create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, + int64_t n_ff_, int64_t n_expert_, int flags); +diff --git a/llama.cpp/src/llama.cpp b/llama.cpp/src/llama.cpp +--- a/llama.cpp/src/llama.cpp ++++ b/llama.cpp/src/llama.cpp +@@ -453,6 +453,120 @@ + return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params); + } + ++// opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant load surface ++static bool llama_mtp_vocab_matches(const llama_model & tgt, const llama_model & aux) { ++ const llama_vocab & vt = tgt.vocab; ++ const llama_vocab & va = aux.vocab; ++ if (vt.n_tokens() != va.n_tokens()) { ++ LLAMA_LOG_ERROR("%s: vocab size mismatch (target=%u assistant=%u)\n", __func__, vt.n_tokens(), va.n_tokens()); ++ return false; ++ } ++ // opencoti F5 M6-S4 mtp: the gemma4_assistant draft head ships NO tokenizer of its own ++ // (vocab type NONE) — it shares the target's vocab, and its token_ordering/centroids are sized ++ // to the shared vocab. token_get_text() asserts on a NONE vocab (no id_to_token table), so the ++ // per-token text comparison below is both impossible and meaningless for the vocab-less draft. ++ // The n_tokens() equality above is the meaningful invariant; accept once it holds. ++ if (va.get_type() == LLAMA_VOCAB_TYPE_NONE) { ++ return true; ++ } ++ constexpr int32_t k_check_from = 5; // align with speculative MTP vocab check ++ for (uint32_t i = (uint32_t) k_check_from; i < vt.n_tokens(); ++i) { ++ const llama_token id = (llama_token) i; ++ if (std::strcmp(vt.token_get_text(id), va.token_get_text(id)) != 0) { ++ LLAMA_LOG_ERROR("%s: vocab text mismatch at token id %d\n", __func__, (int) id); ++ return false; ++ } ++ } ++ return true; ++} ++ ++int llama_model_load_mtp_from_file(struct llama_model * model, const char * path_mtp, struct llama_model_params params) { ++ if (!model || !path_mtp || !path_mtp[0]) { ++ LLAMA_LOG_ERROR("%s: invalid arguments\n", __func__); ++ return -1; ++ } ++ ++ llama_model * tgt = (llama_model *) model; ++ if (tgt->arch != LLM_ARCH_GEMMA4) { ++ LLAMA_LOG_ERROR("%s: MTP target must be arch gemma4 (got %s)\n", __func__, llm_arch_name(tgt->arch)); ++ return -2; ++ } ++ ++ llama_model * aux = llama_model_load_from_file(path_mtp, params); ++ if (!aux) { ++ LLAMA_LOG_ERROR("%s: failed to load assistant from '%s'\n", __func__, path_mtp); ++ return -3; ++ } ++ ++ if (aux->arch != LLM_ARCH_GEMMA4_ASSISTANT) { ++ LLAMA_LOG_ERROR("%s: MTP weights must be arch gemma4-assistant (got %s)\n", __func__, llm_arch_name(aux->arch)); ++ llama_model_free(aux); ++ return -4; ++ } ++ ++ // opencoti F5 M6-S4 mtp: arch prefix now hyphenated (upstream-aligned "gemma4-assistant"); the ++ // GGUF key prefix follows the arch string, so this raw lookup must match. ++ const auto req_it = aux->gguf_kv.find("gemma4-assistant.requires_target_arch"); ++ if (req_it != aux->gguf_kv.end() && req_it->second != "gemma4") { ++ LLAMA_LOG_ERROR("%s: assistant requires_target_arch='%s' (expected gemma4)\n", __func__, req_it->second.c_str()); ++ llama_model_free(aux); ++ return -5; ++ } ++ ++ if (aux->hparams.n_embd_out_impl == 0 || aux->hparams.n_embd_out_impl != tgt->hparams.n_embd) { ++ LLAMA_LOG_ERROR("%s: assistant backbone width (embedding_length_out) %u must match target n_embd %u\n", __func__, ++ aux->hparams.n_embd_out_impl, tgt->hparams.n_embd); ++ llama_model_free(aux); ++ return -6; ++ } ++ ++ if (!llama_mtp_vocab_matches(*tgt, *aux)) { ++ llama_model_free(aux); ++ return -7; ++ } ++ // opencoti F5 M6-S4 mtp: namespace ALL assistant tensors under an in-memory "mtp." prefix so -ot ++ // rules can target them without colliding with the target model's identically-named generic tensors ++ // (the assistant has its own blk.*, token_embd, output_norm). This is a post-load in-memory rename ++ // for -ot ONLY; it does NOT touch GGUF identity — on-disk names stay upstream-aligned (nextn.* / ++ // masked_embd_*, none of which start with "mtp."), so every entry is prefixed exactly once and the ++ // existing "mtp.*" wildcard -ot rules keep matching. ++ for (auto & kv : aux->tensors_by_name) { ++ if (kv.first.substr(0, 4) != "mtp.") { ++ std::string new_name = "mtp." + kv.first; ++ ggml_set_name(kv.second, new_name.c_str()); ++ kv.first = new_name; ++ } ++ } ++ tgt->mtp_assistant.reset(aux); ++ return 0; ++} ++ ++const struct llama_model * llama_model_get_mtp_assistant(const struct llama_model * model) { ++ if (!model) { ++ return nullptr; ++ } ++ const llama_model * m = (const llama_model *) model; ++ return m->mtp_assistant.get(); ++} ++ ++bool llama_model_has_mtp_assistant(const struct llama_model * model) { ++ return llama_model_get_mtp_assistant(model) != nullptr; ++} ++ ++// opencoti F5 M6-S4 mtp: fork-private C API (no upstream counterpart — upstream uses ctx_other and ++// never exposes the assistant backbone width). Name kept for ABI stability; the backing hparams field ++// is now n_embd_out_impl (upstream-aligned, from %s.embedding_length_out). ++uint32_t llama_model_mtp_n_embd_backbone(const struct llama_model * model) { ++ if (!model) { ++ return 0; ++ } ++ const llama_model * m = (const llama_model *) model; ++ if (!m->mtp_assistant) { ++ return 0; ++ } ++ return m->mtp_assistant->hparams.n_embd_out_impl; ++} ++ + void llama_model_save_to_file(const struct llama_model * model, const char * path_model) { + llama_model_saver ms(model); + ms.add_kv_from_model(); +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -132,6 +132,14 @@ + } + + std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { ++ // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), ++ // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF ++ // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ ++ // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. ++ if (params.gtype == LLM_GRAPH_TYPE_MTP) { ++ GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); ++ return std::make_unique(*this, *mtp_assistant, params); ++ } + return std::make_unique(*this, params); + } + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -809,6 +809,26 @@ + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; + }; + ++// opencoti F5 M6-S4 mtp ++// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. ++struct llm_build_gemma4_mtp : public llm_graph_context { ++ const llama_model & target; ++ const llama_model & mtp; ++ ++ llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); ++}; ++ ++// opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant — loaded INTO a gemma4 target via ++// llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own ++// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the ++// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). ++struct llama_model_gemma4_assistant : public llama_model_base { ++ llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} ++ void load_arch_hparams(llama_model_loader & ml) override; ++ void load_arch_tensors(llama_model_loader & ml) override; ++ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; ++}; ++ + + struct llama_model_gemma_embedding : public llama_model_base { + llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {} +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -806,7 +806,19 @@ + const bool spec_mtp = std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end(); +- const bool has_draft = params_base.speculative.has_dft(); ++ // opencoti F5 M6-S4 mtp (#485): the Gemma assistant draft (COMMON_SPECULATIVE_TYPE_MTP) is a ++ // separate GGUF (so has_dft()==true) but CANNOT be loaded standalone — gemma4-assistant's ++ // build_arch_graph throws "cannot be used as a primary model". The old code fell into the ++ // has_draft measure branch below, called common_get_device_memory_data() on it, threw, and ++ // logged a misleading "[spec] failed to measure draft model memory" (it reads as a boot error ++ // and tripped probe readiness loops -> false BOOT-FAIL, the bug-550 misdiagnosis). The ++ // assistant head loads INTO the target model post-fit and is small, so skip the standalone ++ // reserve entirely for the assistant engine (the caught-exception path already added 0 bytes; ++ // this just makes it explicit and silences the bogus error). ++ const bool spec_assistant_fit = std::find(params_base.speculative.types.begin(), ++ params_base.speculative.types.end(), ++ COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); ++ const bool has_draft = params_base.speculative.has_dft() && !spec_assistant_fit; + + if (has_draft || spec_mtp) { + common_params params_dft = params_base; +@@ -906,7 +918,58 @@ + + add_bos_token = llama_vocab_get_add_bos(vocab); + +- if (params_base.speculative.has_dft()) { ++ // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so ++ // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file — ++ // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at ++ // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from ++ // both native branches (separate-model draft below; NextN MTP-ctx-against-target further ++ // down). Detect it via the enabled types vector, load into the target, and leave ++ // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's ++ // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). ++ const bool spec_assistant = std::find(params_base.speculative.types.begin(), ++ params_base.speculative.types.end(), ++ COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); ++ if (params_base.speculative.has_dft() && spec_assistant) { ++ // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target ++ // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the ++ // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout ++ // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. ++ // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 ++ // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it ++ // explicitly and fail clean at boot instead of crashing mid-request. ++ if (params_base.n_parallel > 1 && !params_base.kv_unified) { ++ SRV_ERR("%s", ++ "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " ++ "--kv-unified: the per-stream-split KV layout is not supported by the single-context " ++ "MTP cross-read. Re-run with --kv-unified (PolyKV already sets it).\n"); ++ return false; ++ } ++ ++ const auto & params_spec = params_base.speculative.draft; ++ ++ SRV_INF("loading MTP assistant '%s' into target\n", params_spec.mparams.path.c_str()); ++ ++ auto params_dft = params_base; ++ params_dft.devices = params_spec.devices; ++ params_dft.model = params_spec.mparams; ++ params_dft.n_gpu_layers = params_spec.n_gpu_layers; ++ params_dft.cache_type_k = params_spec.cache_type_k; ++ params_dft.cache_type_v = params_spec.cache_type_v; ++ params_dft.tensor_buft_overrides = params_spec.tensor_buft_overrides; ++ ++ auto mparams_dft = common_model_params_to_llama(params_dft); ++ ++ const int mtp_rc = llama_model_load_mtp_from_file(model_tgt, params_dft.model.path.c_str(), mparams_dft); ++ if (mtp_rc != 0) { ++ SRV_ERR("failed to load MTP assistant '%s' (rc=%d)\n", params_dft.model.path.c_str(), mtp_rc); ++ return false; ++ } ++ ++ // No separate draft model/context: the assistant decodes on ctx_tgt via llama_decode_mtp. ++ params_base.speculative.draft.ctx_tgt = ctx_tgt; ++ params_base.speculative.draft.ctx_dft = nullptr; ++ SRV_INF("%s", "MTP assistant loaded into target\n"); ++ } else if (params_base.speculative.has_dft()) { + // TODO speculative: move to common/speculative.cpp? + const auto & params_spec = params_base.speculative.draft; + +diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -0,0 +1,488 @@ ++// opencoti F5 M6-S4 mtp ++#include "models.h" ++ ++#include ++#include ++ ++static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { ++ p.arch = mtp_model.arch; ++ p.hparams = mtp_model.hparams; ++ p.gtype = LLM_GRAPH_TYPE_MTP; ++ return p; ++} ++ ++// Last layer in [range_start, range_end) whose attention type matches want_swa. ++static int32_t gemma4_mtp_kv_layer_last_in_range( ++ const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { ++ int32_t best = -1; ++ if (range_start < 0) { ++ range_start = 0; ++ } ++ if (range_end > (int32_t) tgt.n_layer) { ++ range_end = (int32_t) tgt.n_layer; ++ } ++ for (int32_t il = range_start; il < range_end; ++il) { ++ if (tgt.is_swa((uint32_t) il) == want_swa) { ++ best = il; ++ } ++ } ++ return best; ++} ++ ++// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> ++// (post-projected backbone hidden, vocabulary logits, argmax token id). ++// ++// Used by both the single-step path (n_mtp_steps==1) and the chained path ++// (n_mtp_steps>1, called n_mtp_steps times within the same graph). ++// ++// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position ++// for this step's RoPE (the cross-attn mask is shared across steps because the ++// target's KV cache only contains positions ≤ attn_pos and all step positions ++// are > attn_pos, so causal/SWA admit all target cells uniformly). ++// ++// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too — required for greedy match with verify). ++// `out_argmax` (I32 [1]): greedy token id on-device. ++static void gemma4_mtp_build_one_step( ++ const llm_graph_context & gctx, ++ const llama_model & target, ++ const llama_model & mtp, ++ llm_graph_input_attn_kv_iswa * inp_attn, ++ ggml_tensor * tok_step, // I32 [1] ++ ggml_tensor * h_step, // F32 [n_bb, 1] ++ ggml_tensor * pos_step, // I32 [1] ++ ggml_tensor ** out_logits, // F32 [n_vocab, 1] ++ ggml_tensor ** out_h_post, // F32 [n_bb, 1] ++ ggml_tensor ** out_argmax) { // I32 [1] ++ auto * ctx0 = gctx.ctx0; ++ auto * gf = gctx.gf; ++ const auto & hparams = gctx.hparams; ++ const auto & cparams = gctx.cparams; ++ const int n_layer = (int) gctx.n_layer; ++ const int n_tokens = (int) gctx.n_tokens; ++ const int n_ctx_orig = (int) gctx.n_ctx_orig; ++ const int rope_type = gctx.rope_type; ++ const float ext_factor = gctx.ext_factor; ++ const float attn_factor = gctx.attn_factor; ++ const float beta_fast = gctx.beta_fast; ++ const float beta_slow = gctx.beta_slow; ++ auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; ++ ++ const float tok_scale = sqrtf((float) target.hparams.n_embd); ++ ++ ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); ++ cb(tok_e, "mtp_tgt_tok_embd", -1); ++ ++ // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). ++ // Use target n_embd so Edge / non-Edge targets match the main forward. ++ tok_e = ggml_scale(ctx0, tok_e, tok_scale); ++ cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); ++ ++ ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); ++ cb(inp_cat, "mtp_concat", -1); ++ ++ ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); ++ cb(inpL, "mtp_pre_proj_out", -1); ++ ++ ggml_build_forward_expand(gf, inpL); ++ ++ ggml_tensor * cur = nullptr; ++ ++ for (int il = 0; il < n_layer; ++il) { ++ const int64_t n_embd_head = hparams.n_embd_head_k(il); ++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); ++ ++ const int64_t n_head = hparams.n_head(il); ++ ++ const float freq_base_l = mtp.get_rope_freq_base(cparams, il); ++ const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); ++ const int n_rot_l = hparams.n_rot(il); ++ ++ cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "attn_norm", il); ++ ++ ggml_tensor * freq_factors = nullptr; ++ if (!hparams.is_swa(il)) { ++ freq_factors = mtp.layers[il].rope_freqs; ++ } ++ ++ ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); ++ cb(Qcur, "Qcur", il); ++ ++ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); ++ ++ Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); ++ cb(Qcur, "Qcur_normed", il); ++ ++ Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++ cb(Qcur, "Qcur_pos", il); ++ ++ const bool read_swa = hparams.is_swa(il); ++ ++ const int32_t n_tgt = (int32_t) target.hparams.n_layer; ++ ++ // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads ++ // ONE shared KV per attention type from the target — the LAST layer of that type. ++ // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py ++ // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} ++ const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); ++ ++ GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); ++ ++ // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and ++ // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the ++ // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must ++ // ALWAYS fetch V from the cache — not reuse the post-RoPE K tensor. ++ const bool use_k_as_v = false; ++ ++ const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); ++ const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); ++ ++ cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, ++ hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); ++ ++ cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "attn_post_norm", il); ++ ++ ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); ++ cb(attn_out, "attn_out", il); ++ ++ GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); ++ ++ cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "ffn_norm", il); ++ ++ cur = gctx.build_ffn(cur, ++ mtp.layers[il].ffn_up, nullptr, nullptr, ++ mtp.layers[il].ffn_gate, nullptr, nullptr, ++ mtp.layers[il].ffn_down, nullptr, nullptr, ++ nullptr, ++ LLM_FFN_GELU, LLM_FFN_PAR, il); ++ cb(cur, "ffn_out", il); ++ ++ cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "ffn_post_norm", il); ++ ++ cur = ggml_add(ctx0, cur, attn_out); ++ ++ if (mtp.layers[il].out_scale) { ++ cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); ++ cb(cur, "out_scaled", il); ++ } ++ ++ cur = gctx.build_cvec(cur, il); ++ cb(cur, "l_out", il); ++ ++ inpL = cur; ++ } ++ ++ cur = inpL; ++ ++ cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "result_norm", -1); ++ ++ ggml_tensor * h_inner = cur; ++ ++ ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); ++ cb(backbone, "mtp_post_proj_out", -1); ++ ++ const int64_t n_vocab = mtp.tok_embd->ne[1]; ++ const int64_t n_tokens_mtp = h_inner->ne[1]; ++ ++ if (mtp.hparams.use_ordered_embeddings) { ++ // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full ++ // [n_vocab] row then argmax — matches masked full-vocab greedy (sparse-only argmax broke server accept). ++ GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); ++ GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); ++ const uint32_t n_c = mtp.hparams.n_centroids; ++ const uint32_t top_k = mtp.hparams.centroid_top_k; ++ GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); ++ GGML_ASSERT(n_vocab % (int64_t) n_c == 0); ++ const int64_t vsc = n_vocab / (int64_t) n_c; ++ ++ ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); ++ cb(centroid_logits, "mtp_centroid_logits", -1); ++ ++ ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); ++ cb(topk_idx, "mtp_centroid_topk_idx", -1); ++ ++ const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); ++ ggml_tensor * ordering = ggml_view_2d( ++ ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); ++ cb(ordering, "mtp_token_ordering_view", -1); ++ ++ ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); ++ cb(sel_ids, "mtp_selected_token_ids", -1); ++ ++ const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; ++ ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); ++ cb(flat_ids, "mtp_selected_token_ids_flat", -1); ++ ++ ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); ++ cb(sel_emb, "mtp_selected_embd", -1); ++ ++ ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); ++ cb(sel_logits, "mtp_selected_logits", -1); ++ ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); ++ ++ ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); ++ logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); ++ cb(logits_full, "mtp_logits_masked_base", -1); ++ ++ ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); ++ ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); ++ cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); ++ cb(cur, "mtp_logits_scatter_view", -1); ++ cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); ++ cb(cur, "mtp_logits_full", -1); ++ } else { ++ cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); ++ cb(cur, "result_output_dense", -1); ++ } ++ ++ if (hparams.f_final_logit_softcapping) { ++ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); ++ cur = ggml_tanh(ctx0, cur); ++ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); ++ } ++ ++ cb(cur, "result_output", -1); ++ ++ // Greedy argmax on-device: I32 [1] token index into the vocabulary row. ++ ggml_tensor * arg = ggml_argmax(ctx0, cur); ++ cb(arg, "result_argmax", -1); ++ ++ *out_logits = cur; ++ *out_h_post = backbone; ++ *out_argmax = arg; ++} ++ ++llm_build_gemma4_mtp::llm_build_gemma4_mtp( ++ const llama_model & target_model, ++ const llama_model & mtp_model, ++ const llm_graph_params & params) : ++ llm_graph_context(graph_params_for_mtp(params, mtp_model)), ++ target(target_model), ++ mtp(mtp_model) { ++ const int64_t n_bb = mtp.hparams.n_embd_out_impl; ++ GGML_ASSERT(n_bb > 0); ++ GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); ++ ++ // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 ++ // (fused path) chain from the previous step's on-device argmax + post-projection, so only ++ // step 0 reads token/h from the host. ++ ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); ++ ggml_set_input(inp_tok); ++ cb(inp_tok, "mtp_inp_last_token", -1); ++ ++ ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); ++ ggml_set_input(inp_h); ++ cb(inp_h, "mtp_inp_h_prev", -1); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into ++ // ONE graph (no per-step host round-trip / GPU sync — the throughput fix). n_steps==1 is the ++ // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). ++ const int32_t n_steps = std::max(params.n_mtp_steps, 1); ++ ++ auto * inp_attn = build_attn_inp_kv_iswa(); ++ ++ if (n_steps <= 1) { ++ { ++ auto inp_wrap = std::make_unique(); ++ inp_wrap->inp_last_token = inp_tok; ++ inp_wrap->inp_h_prev = inp_h; ++ res->add_input(std::move(inp_wrap)); ++ } ++ ++ ggml_tensor * inp_pos = build_inp_pos(); ++ ++ ggml_tensor * logits = nullptr; ++ ggml_tensor * h_post = nullptr; ++ ggml_tensor * arg = nullptr; ++ gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, ++ inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); ++ ++ res->t_embd = h_post; ++ res->t_logits = logits; ++ res->t_argmax = arg; ++ ++ ggml_build_forward_expand(gf, arg); ++ ggml_build_forward_expand(gf, h_post); ++ return; ++ } ++ ++ // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is ++ // shared across steps — see gemma4_mtp_build_one_step's contract). The chain ties step k+1's ++ // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. ++ auto inp_wrap = std::make_unique(); ++ inp_wrap->inp_last_token = inp_tok; ++ inp_wrap->inp_h_prev = inp_h; ++ inp_wrap->inp_pos_steps.reserve(n_steps); ++ ++ std::vector pos_steps; ++ pos_steps.reserve(n_steps); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); ++ ggml_set_input(p); ++ cb(p, "mtp_inp_pos_step", k); ++ pos_steps.push_back(p); ++ inp_wrap->inp_pos_steps.push_back(p); ++ } ++ res->add_input(std::move(inp_wrap)); ++ ++ ggml_tensor * tok_k = inp_tok; ++ ggml_tensor * h_k = inp_h; ++ ++ std::vector step_args; ++ step_args.reserve(n_steps); ++ ggml_tensor * last_logits = nullptr; ++ ggml_tensor * last_h_post = nullptr; ++ ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * logits_k = nullptr; ++ ggml_tensor * h_post_k = nullptr; ++ ggml_tensor * arg_k = nullptr; ++ gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, ++ tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); ++ ++ step_args.push_back(arg_k); ++ last_logits = logits_k; ++ last_h_post = h_post_k; ++ ++ // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the ++ // target vocab — already used as a get_rows index inside build_one_step), next step's ++ // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. ++ tok_k = arg_k; ++ h_k = h_post_k; ++ } ++ ++ // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 ++ // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend — a ++ // negligible side-consumer that does NOT touch the on-device get_rows chain above. ++ ggml_tensor * all_args = step_args[0]; ++ for (int32_t k = 1; k < n_steps; ++k) { ++ all_args = ggml_concat(ctx0, all_args, step_args[k], 0); ++ } ++ cb(all_args, "mtp_fused_argmax", -1); ++ ++ res->t_argmax = all_args; // I32 [n_steps] — the drafted token block ++ res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed ++ res->t_logits = last_logits; // unused on the live path (out_logits == NULL) ++ ++ ggml_build_forward_expand(gf, all_args); ++ ggml_build_forward_expand(gf, last_h_post); ++} ++ ++// opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via ++// llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path ++// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the ++// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 ++// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. ++void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { ++ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ++ ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); ++ ++ uint32_t n_kv_shared_layers = 0; ++ ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); ++ ++ hparams.n_layer_kv_from_start = hparams.n_layer - (int32_t) n_kv_shared_layers; ++ hparams.f_attention_scale = 1.0f; ++ ++ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); ++ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ++ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); ++ ++ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); ++ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); ++ ++ // opencoti F5 M6-S4 mtp: backbone width via the generic upstream key (#23398: %s.embedding_length_out ++ // → hparams.n_embd_out_impl), replacing our bespoke %s.n_embd_backbone. llama_model::load_hparams also ++ // reads it generically; re-read here so the assistant loader is self-contained. ++ ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, hparams.n_centroids, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, hparams.centroid_top_k, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, hparams.attention_k_eq_v, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, hparams.use_ordered_embeddings, false); ++ ++ type = LLM_TYPE_UNKNOWN; ++} ++ ++void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { ++ LLAMA_LOAD_LOCALS; ++ ++ const uint32_t n_bb = hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ throw std::runtime_error("gemma4-assistant: embedding_length_out (backbone width) must be set in GGUF metadata"); ++ } ++ if (n_embd_head_k != n_embd_head_v) { ++ throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k == n_embd_head_v"); ++ } ++ if (hparams.n_embd_head_k_swa != hparams.n_embd_head_v_swa) { ++ throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k_swa == n_embd_head_v_swa"); ++ } ++ ++ // opencoti F5 M6-S4 mtp (#408/bug-408): the assistant's token_embd is the TIED OUTPUT head (a dense ++ // LM head used only as a build_lora_mm weight, never an input get_rows — the input embedding comes ++ // from the TARGET's tok_embd). The P4 throughput win (#408) routes it to the OUTPUT buft so the ++ // per-step full-vocab mul_mat runs on the GPU instead of a graph-fragmenting CPU island. TOKEN_EMBD ++ // classifies LLM_TENSOR_LAYER_INPUT, so the standard create_tensor would land it on the CPU buft. ++ // create_tensor_output_head() (llama_model_base) passes dev_output.buft_list in the INPUT slot to ++ // force GPU output placement — additive, sole-use accounting (NOT TENSOR_DUPLICATED). This re-claims ++ // the 1.46× the 0.10.3 bump dropped; correctness is byte-identical (placement only). ++ tok_embd = create_tensor_output_head(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); ++ ++ // opencoti F5 M6-S4 mtp: GGUF tensor names upstream-aligned (#23398/#24282) — nextn.* / masked_embd_*. ++ // The C++ member names stay mtp_* (control-plane; our single-context engine, not upstream's ctx_other). ++ mtp_pre_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_PRE, "weight"), {2 * (int64_t) n_bb, n_embd}, 0); ++ mtp_post_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_POST, "weight"), {n_embd, (int64_t) n_bb}, 0); ++ ++ if (hparams.use_ordered_embeddings) { ++ const uint32_t n_c = hparams.n_centroids; ++ if (n_c == 0) { ++ throw std::runtime_error("gemma4-assistant: use_ordered_embeddings requires n_centroids > 0"); ++ } ++ mtp_centroids = create_tensor(tn(LLM_TENSOR_MASKED_EMBD_CENTROIDS, "weight"), {n_embd, (int64_t) n_c}, 0); ++ // upstream loads ordering WITHOUT a .weight suffix → bare "masked_embd_ordering" ++ mtp_token_ordering = create_tensor(tn(LLM_TENSOR_MASKED_EMBD_ORDERING), {(int64_t) n_vocab}, TENSOR_NOT_REQUIRED); ++ } ++ ++ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); ++ ++ int rope_freqs_flag = 0; ++ ++ for (int i = 0; i < n_layer; ++i) { ++ auto & layer = layers[i]; ++ const int64_t n_head = hparams.n_head(i); ++ const int64_t n_embd_head = hparams.n_embd_head_k(i); ++ ++ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); ++ ++ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); ++ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0); ++ ++ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0); ++ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); ++ ++ layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), {1u}, TENSOR_NOT_REQUIRED); ++ ++ if (!hparams.is_swa(i)) { ++ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head/2}, rope_freqs_flag); ++ rope_freqs_flag = TENSOR_DUPLICATED; ++ } ++ ++ const int64_t n_ff_cur = hparams.n_ff(i); ++ ++ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); ++ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff_cur}, 0); ++ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_cur}, 0); ++ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_cur, n_embd}, 0); ++ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); ++ } ++} ++ ++std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params &) const { ++ throw std::runtime_error( ++ "gemma4_assistant cannot be used as a primary model (-m). " ++ "Load the Gemma 4 target with -m, then call llama_model_load_mtp_from_file() with the assistant GGUF."); ++} diff --git a/patches/0080-mtp-d512-fa.patch b/patches/0080-mtp-d512-fa.patch new file mode 100644 index 0000000000000000000000000000000000000000..bf8384e1c172a3b05a0b2dcdc8382231c368a8a0 --- /dev/null +++ b/patches/0080-mtp-d512-fa.patch @@ -0,0 +1,155 @@ +From: opencoti +Subject: [PATCH 0080] Gemma-4 assistant-MTP D512/gqa<=2 flash-attn — real f16 VEC kernel (#491+#493, bug-567) + +The Gemma-4 E4B assistant-MTP drafter's single GLOBAL (non-SWA) layer is head_count 4 / +head_count_kv 2 -> gqa_ratio 2 at head_dim 512. On the 0.10.3 base NEITHER the MMA +(switch_ncols2, fattn.cu) NOR the TILE (switch_ncols2, fattn-tile.cuh) D512 path instantiates +ncols2 in {1,2} (gated DV<=256 for binary size), so a D512/gqa<=2 flash-attn op aborts at +runtime (bug-567). Every other Gemma-4 config takes ncols2 in {4,8} and is unaffected +(A4B/12B/31B and the E4B base are gqa>=4; A4B globals are head16/kv2 -> gqa 8). + +#491 (host stop-gap): a guard in build_attn_mha (llama-graph.cpp) routed the D512/gqa<=2 layer +to the EXISTING non-FA explicit-softmax branch. Correct but not a real flash-attn — "fa off" on +the hot path. + +#493 (this patch, the real fix): add a REAL D512 f16 flash-attn via the VEC kernel, and narrow +the #491 guard so f16/f32 K/V take it; the guard now fires ONLY for the residual with no D512 +kernel (quantized / bf16 K/V), which the assistant drafter never hits. + - template-instances/fattn-vec-instance-f16-f16.cu + generate_cu_files.py: + DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16) instance (the autogen line + the file). + - ggml-cuda/fattn-vec.cuh: extern decl for the D512 f16 instance. + - ggml-cuda/fattn.cu: (a) selector route in ggml_cuda_get_best_fattn_kernel — D512 && gqa_ratio<=2 + && gqa_opt_applies && K/V f16 -> BEST_FATTN_KERNEL_VEC; (b) the matching VEC dispatch case + FATTN_VEC_CASE(512, F16, F16) in ggml_cuda_flash_attn_ext_vec, BEFORE the GGML_ABORT. + NOTE: fattn.cu has TWO independent head-size switches (selector + VEC dispatch); both need the + D512 case or the selector picks VEC but the dispatch falls through to a decode-time abort. + - llama.cpp/src/llama-graph.cpp: narrow the #491 guard — fa_no_gpu_kernel fires only when K/V are + NOT f16/f32 (fa_d512_f16_vec), so f16 D512/gqa<=2 now flash-attns on-GPU. + +The VEC kernel tiles Q->ne[1] (cols_per_block 1 decode / 2 prefill+verify) so the one f16 D512 +instance is a real flash-attn at ANY n_q (covers decode, prefill, and the spec-verify batch). +logit_softcap is forced off for D!=128,256 in the VEC kernel; safe because Gemma-4 attn_soft_cap +is false (softcap 0 -> use_logit_softcap false). + +Gate (.opencoti/m493-fa-d512-gate.sh): E4B -fa on assistant-MTP decodes 147.86 tps with NO abort; +the verified-greedy output is BYTE-IDENTICAL to the #491 non-FA soft_max path and draft acceptance +is unchanged (0.67910 both), so the new VEC FA is numerically correct (exact spec-decode validates +every token, and a wrong FA would collapse acceptance). A4B (gqa 8, untouched MMA path) regresses +nowhere. Both DSO paths rebuilt byte-identical (sha256 a8ddcd82). + +Marker convention: every changed file carries an `opencoti #491`/`opencoti #493` marker; see +docs/protocols/UPSTREAM_SYNC.md. All hunks are nested (llama.cpp/); no outer-vendor delta. + +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2122,7 +2122,29 @@ + + ggml_tensor * cur; + +- const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr; ++ // opencoti F5 M6-S4 mtp (#441/#491): head_dim 512 has NO GPU flash-attn kernel for gqa_ratio ≤ 2. ++ // Both the MMA (ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2, fattn.cu) and TILE ++ // (launch_fattn_tile_switch_ncols2, fattn-tile.cuh) D512 paths only instantiate ncols2 ∈ {4,8} ++ // (gqa_ratio > 2); the ncols2 ∈ {1,2} variants are gated out for binary size (DV<=256 only), so a ++ // D512/gqa≤2 flash-attn op aborts at runtime (fattn.cu:132 / fattn-tile.cuh:1322). That is exactly ++ // the Gemma-4 E4B assistant-MTP drafter's single GLOBAL layer: head_count 4 / head_count_kv 2 → ++ // gqa_ratio 2. Route it through the EXISTING non-FA explicit-softmax branch below (any head_dim/gqa ++ // — see the "non-flash-attn branch" fallback note at the top of this fn). q/k here are post-permute ++ // (flash-attn layout), so q->ne[2]/k->ne[2] == the CUDA kernel's gqa_ratio and k->ne[0] == head_dim. ++ // NARROW: every working D512 model (A4B/E2B/12B/31B + the E4B base, gqa_ratio ≥ 4) keeps flash-attn, ++ // byte-unchanged; only the gqa≤2 case is special. ++ // opencoti #493 (F5 M6-S4): D512/gqa≤2 with f16 (or f32→f16-cast) K AND V now has a REAL GPU ++ // flash-attn — the f16 D512 VEC instance, which tiles Q->ne[1] (cols_per_block 1 decode / 2 ++ // prefill+verify) so it computes at ANY n_q (see ggml_cuda_get_best_fattn_kernel, fattn.cu). So ++ // -fa on COMPUTES on-GPU for the E4B drafter (its KV is f16) instead of diverting to soft_max; no ++ // global fa-off, no non-FA fallback for the real path. The guard now fires ONLY for the residual ++ // with no D512/gqa≤2 kernel: QUANTIZED or bf16 K/V (the VEC D512 instance is f16-only; MMA/TILE ++ // abort, bug-567) — an exotic config the assistant drafter never hits. f16/f32 K/V → flash-attn. ++ const bool fa_d512_f16_vec = (k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32) && ++ (v->type == GGML_TYPE_F16 || v->type == GGML_TYPE_F32); ++ const bool fa_no_gpu_kernel = k->ne[0] == 512 && k->ne[2] > 0 && (q->ne[2] / k->ne[2]) <= 2 && ++ !fa_d512_f16_vec; ++ const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && !fa_no_gpu_kernel; + if (use_flash_attn) { + GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet"); + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -360,6 +360,13 @@ + FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0) + FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) + ++ // opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC dispatch — the runtime counterpart to the ++ // selector's D512/gqa<=2 f16 VEC route (ggml_cuda_get_best_fattn_kernel) + the f16-f16 D512 ++ // instance. Without this case the selector picks VEC but the dispatch falls through to the abort ++ // below (bug-567 reappeared as a *decode-time* fatal at fattn.cu:363). The macro accepts F16 and ++ // F32→F16 for both K and V, matching the graph guard. Gemma-4 E4B assistant-MTP drafter global layer. ++ FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16) ++ + GGML_ABORT("fatal error"); + } + +@@ -513,6 +520,20 @@ + return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE; + } + ++ // opencoti #493 (F5 M6-S4): D512 (Gemma-4 GLOBAL layer) at gqa_ratio <= 2 — the E4B assistant-MTP ++ // drafter (head_count 4 / head_count_kv 2 → gqa 2) — has NO MMA/TILE kernel: switch_ncols2 aborts ++ // for DKQ>256 && gqa<=2 (the ncols2 ∈ {1,2} D512 instances are if-constexpr'd out, bug-567). The ++ // f16 D512 VEC instance (#493) tiles Q->ne[1] (cols_per_block 1 for decode, 2 for prefill/verify — ++ // ggml_cuda_flash_attn_ext_vec_case) so it is a REAL flash-attn at ANY n_q. Route f16/f16 D512 here ++ // when gqa<=2 so -fa on COMPUTES instead of diverting to the #491 non-FA host guard. D512/gqa>2 ++ // (A4B / 12B / 31B / E4B base) keeps the faster MMA ncols2 ∈ {4,8} path — byte-unchanged. Past the ++ // case-512 check above, gqa_opt_applies already holds (so gqa_ratio>=2); gqa_ratio<=2 ⇒ exactly 2. ++ if (Q->ne[0] == 512 && gqa_ratio <= 2 && gqa_opt_applies && ++ K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && ++ K->ne[1] % FATTN_KQ_STRIDE == 0) { ++ return BEST_FATTN_KERNEL_VEC; ++ } ++ + // If Turing tensor cores are available, use them: + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -631,3 +631,8 @@ + // layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256. + extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); + extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); ++ ++// opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC — Gemma-4 GLOBAL layer at gqa_ratio<=2 (the E4B ++// assistant-MTP drafter), where MMA/TILE abort (DKQ>256 && gqa<=2, bug-567). The VEC kernel tiles ++// Q->ne[1] so it is a real flash-attn at ANY n_q. Definition in fattn-vec-instance-f16-f16.cu. ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +@@ -5,3 +5,4 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16); // opencoti #493: Gemma-4 GLOBAL head_dim 512 f16, gqa<=2 +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +@@ -74,6 +74,17 @@ + with open(f"fattn-vec-instance-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f: + f.write(SOURCE_FATTN_VEC.format(type_k=type_k, type_v=type_v)) + ++# opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC for Gemma-4 GLOBAL layers at gqa_ratio<=2 (the E4B ++# assistant-MTP drafter's single global layer is head_count 4 / head_count_kv 2 → gqa 2). The MMA and ++# TILE D512 paths only instantiate ncols2 ∈ {4,8} (gqa>2); ncols2 ∈ {1,2} are if-constexpr'd out, so a ++# D512/gqa<=2 op aborts at runtime (fattn.cu switch_ncols2, bug-567). The VEC kernel tiles Q->ne[1] ++# (cols_per_block 1 for decode, 2 for prefill/verify — ggml_cuda_flash_attn_ext_vec_case) so it is a ++# REAL flash-attn at ANY n_q for D512. Mirrors #411's turbo512 but for plain f16 K/V → -fa on COMPUTES ++# instead of falling back to the #491 non-FA host guard. f16/f16 only (drafter KV is f16); quantized ++# D512/gqa<=2 keeps the guard (no VEC D512 quant instance). ++with open("fattn-vec-instance-f16-f16.cu", "a") as f: ++ f.write("DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16); // opencoti #493: Gemma-4 GLOBAL head_dim 512 f16, gqa<=2\n") ++ + # opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec instances + # (K==V same tier only; the in-register readers live in fattn-turbo.cuh). + for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]: diff --git a/patches/0081-tcq-q6-asym-kv.patch b/patches/0081-tcq-q6-asym-kv.patch new file mode 100644 index 0000000000000000000000000000000000000000..1ca9ad0c27b631e4692b5725fdc253cfe37f526d --- /dev/null +++ b/patches/0081-tcq-q6-asym-kv.patch @@ -0,0 +1,2950 @@ +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +index b678d3a3b..17f30f784 100644 +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -421,7 +421,7 @@ $(UI_CPP_GEN) $(UI_H_GEN) &: $(UI_EMBED_TOOL) $(UI_ASSETS_FILES) + # Tool source files + TOOL_QUANTIZE_SRCS := llama.cpp/tools/quantize/quantize.cpp + TOOL_IMATRIX_SRCS := llama.cpp/tools/imatrix/imatrix.cpp +-TOOL_PERPLEXITY_SRCS := llama.cpp/tools/perplexity/perplexity.cpp ++TOOL_PERPLEXITY_SRCS := llama.cpp/tools/perplexity/perplexity.cpp llama.cpp/tools/perplexity/main.cpp + TOOL_BENCH_SRCS := llama.cpp/tools/llama-bench/llama-bench.cpp + + TOOL_SERVER_SRCS := \ +@@ -622,8 +622,12 @@ o/$(MODE)/llama.cpp/imatrix/imatrix: \ + + o/$(MODE)/llama.cpp/perplexity/perplexity: \ + $(TOOL_PERPLEXITY_OBJS) \ ++ $(TOOL_LLAMAFILE_OBJS) \ ++ $(HTTPLIB_OBJS) \ + $$(TINYBLAS_CPU_OBJS) \ + o/$(MODE)/llama.cpp/llama.cpp.a ++ @mkdir -p $(dir $@) ++ $(LINK.o) $(TOOL_PERPLEXITY_OBJS) $(TOOL_LLAMAFILE_OBJS) $(HTTPLIB_OBJS) $(TINYBLAS_CPU_OBJS) o/$(MODE)/llama.cpp/llama.cpp.a $(LOADLIBES) $(LDLIBS) -o $@ + + o/$(MODE)/llama.cpp/llama-bench/llama-bench: \ + $(TOOL_BENCH_OBJS) \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +index a5e6c9182..cfd3b293c 100644 +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -412,11 +412,17 @@ const std::vector kv_cache_types = { + GGML_TYPE_IQ4_NL, + GGML_TYPE_Q5_0, + GGML_TYPE_Q5_1, ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — -ctk/-ctv q6_0. See docs/features/poly_kv.md. ++ GGML_TYPE_Q6_0, + // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2): -ctk/-ctv turbo2|turbo3|turbo4|turbo8 + GGML_TYPE_TURBO2_0, + GGML_TYPE_TURBO3_0, + GGML_TYPE_TURBO4_0, + GGML_TYPE_TURBO8_0, ++ // opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV tiers (buun-aligned names): ++ // -ctk/-ctv turbo3_tcq|turbo2_tcq. See docs/evaluations/context.md §5. ++ GGML_TYPE_TURBO3_TCQ, ++ GGML_TYPE_TURBO2_TCQ, + }; + + static ggml_type kv_cache_type_from_str(const std::string & s) { +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +index 3ab9e7775..3c72dd4b9 100644 +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -437,7 +437,18 @@ extern "C" { + GGML_TYPE_TQ4_1S = 46, // TurboQuant 4-bit weight: WHT-rotated Lloyd-Max, block=32 + // opencoti-hook: turboquant tier family (M6-S2) — 8-bit near-lossless KV rung + GGML_TYPE_TURBO8_0 = 47, // TurboQuant 8-bit KV: WHT + uniform int8 (SCALE=256) +- GGML_TYPE_COUNT = 48, ++ // opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV tiers, ported from buun ++ // (github.com/spiritbuun/buun-llama-cpp). buun uses enum 45/46, but those are our ++ // TQ3_1S/TQ4_1S, so we take the free slots 48/49. KV cache types are string-selected at ++ // runtime (-ctk/-ctv) and NEVER GGUF-serialized, so the type NAME ("turbo3_tcq"/ ++ // "turbo2_tcq") — not the enum value — is what must stay buun-compatible. See context.md §5. ++ GGML_TYPE_TURBO3_TCQ = 48, // TCQ 3-bit KV: FWHT + Viterbi (512-state, k=3, L=9), O(1) decode, 3.25 bpv ++ GGML_TYPE_TURBO2_TCQ = 49, // TCQ 2-bit KV: FWHT + Viterbi (256-state, k=2, L=8), O(1) decode, 2.25 bpv ++ // opencoti-hook: q6_0 KV — see docs/evaluations/context.md §5. Ported from Anbeeld/beellama.cpp ++ // (ik_llama lineage); the anbeeld.com KV-quant benchmark's recommended high-end V tier (q8_0/q6_0). ++ // KV-only (never GGUF-serialized); signed 6-bit, QK6_0=32, 6.5 bpv. ++ GGML_TYPE_Q6_0 = 50, ++ GGML_TYPE_COUNT = 51, + }; + + // precision +diff --git a/llama.cpp/ggml/src/ggml-common.h b/llama.cpp/ggml/src/ggml-common.h +index 82d416e81..551fac0be 100644 +--- a/llama.cpp/ggml/src/ggml-common.h ++++ b/llama.cpp/ggml/src/ggml-common.h +@@ -245,6 +245,17 @@ typedef struct { + } block_q8_0; + static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding"); + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port, ik_llama lineage) — see docs/evaluations/context.md §5. ++// Signed 6-bit: qs[16] holds the low 4 bits of all 32 values; qh[8] holds the high 2 bits packed 4-per-byte. ++// 2 + 8 + 16 = 26 bytes / 32 values = 6.5 bpv. KV-only (never GGUF-serialized). ++#define QK6_0 32 ++typedef struct { ++ ggml_half d; // delta ++ uint8_t qh[QK6_0/4]; // high 2 bits of quants (packed 4 values/byte) ++ uint8_t qs[QK6_0/2]; // low 4 bits / quants ++} block_q6_0; ++static_assert(sizeof(block_q6_0) == sizeof(ggml_half) + QK6_0/4 + QK6_0/2, "wrong q6_0 block size/padding"); ++ + #define QK8_1 32 + typedef struct { + GGML_EXTENSION union { +@@ -531,6 +542,33 @@ typedef struct { + } block_turbo8_0; // 130 bytes total + static_assert(sizeof(block_turbo8_0) == sizeof(ggml_half) + QK_TURBO8, "wrong turbo8_0 block size/padding"); + ++// opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV, ported from buun ++// (github.com/spiritbuun/buun-llama-cpp). Real encode (FWHT + Viterbi) is the CUDA SET_ROWS ++// kernel; real decode (O(1) sliding-window codebook lookup) is in the FA-VEC kernel. CPU ++// to_float/from_float are deliberate stubs (norm-only / zero-fill). block == rotation group == 128. ++// ++// TurboQuant 3-bit TCQ: Trellis-Coded Quantization (right-shift bitshift trellis, k=3, L=9) ++// One block = one 128-element rotation group. Bitstream: 6 zero-prefix + 128×3-bit = 390 bits = 49 bytes. ++// Decode: state_t = read_9_bits(qs, t*3), recon_t = codebook[state_t] * norm. 3.25 bpv → 5.0× vs fp16. ++#define QK_TURBO3_TCQ 128 ++typedef struct { ++ ggml_half norm; // 2 bytes: corrected group L2 norm ++ uint8_t qs[49]; // 49 bytes: 390-bit trellis bitstream (2 padding bits) ++ uint8_t pad; // 1 byte: alignment padding ++} block_turbo3_tcq; // 52 bytes total for 128 values (3.25 bpv) ++static_assert(sizeof(block_turbo3_tcq) == sizeof(ggml_half) + 50, "wrong turbo3_tcq block size/padding"); ++ ++// TurboQuant 2-bit TCQ: Trellis-Coded Quantization (right-shift bitshift trellis, k=2, L=8) ++// One block = one 128-element rotation group. Bitstream: 6 prefix + 128×2-bit = 262 bits = 33 bytes. ++// Decode: state_t = read_8_bits(qs, t*2), recon_t = codebook[state_t] * norm. 2.25 bpv → 7.1× vs fp16. ++#define QK_TURBO2_TCQ 128 ++typedef struct { ++ ggml_half norm; // 2 bytes: corrected group L2 norm ++ uint8_t qs[33]; // 33 bytes: 262-bit trellis bitstream (2 padding bits) ++ uint8_t pad; // 1 byte: alignment padding ++} block_turbo2_tcq; // 36 bytes total for 128 values (2.25 bpv) ++static_assert(sizeof(block_turbo2_tcq) == sizeof(ggml_half) + 34, "wrong turbo2_tcq block size/padding"); ++ + // TQ3_1S: WHT-rotated 3-bit weight quantization (8-level Lloyd-Max for N(0,1)) + // Block size 32, dual half-block scales (d0 for [0..15], d1 for [16..31]) + // Per block: d0(fp16) + d1(fp16) + 3-bit indices packed (12 bytes) = 16 bytes per 32 values +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +index aad25a6ce..468f762db 100644 +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -268,6 +268,11 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { + .vec_dot_type = GGML_TYPE_Q8_1, + .nrows = 1, + }, ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — KV-only scalar 6-bit; from_float via the ++ // ref (no CPU vec_dot — dequant-on-lift converts q6_0→f16 before any matmul). See docs/features/poly_kv.md. ++ [GGML_TYPE_Q6_0] = { ++ .from_float = (ggml_from_float_t) quantize_row_q6_0_ref, ++ }, + [GGML_TYPE_Q8_0] = { + .from_float = quantize_row_q8_0, + .vec_dot = ggml_vec_dot_q8_0_q8_0, +@@ -433,6 +438,15 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { + [GGML_TYPE_TURBO8_0] = { + .from_float = (ggml_from_float_t) quantize_row_turbo8_0_ref, + }, ++ // opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV. CPU from_float is a norm-only ++ // stub (real Viterbi encode is the CUDA set_rows kernel); registered so the dequant-on-lift / ++ // POSITION_WINDOW host-pinned tail set_rows resolves a from_float for the type. ++ [GGML_TYPE_TURBO3_TCQ] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo3_tcq_ref, ++ }, ++ [GGML_TYPE_TURBO2_TCQ] = { ++ .from_float = (ggml_from_float_t) quantize_row_turbo2_tcq_ref, ++ }, + }; + + const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) { +diff --git a/llama.cpp/ggml/src/ggml-cpu/quants.h b/llama.cpp/ggml/src/ggml-cpu/quants.h +index 950b2a916..922db2508 100644 +--- a/llama.cpp/ggml/src/ggml-cpu/quants.h ++++ b/llama.cpp/ggml/src/ggml-cpu/quants.h +@@ -24,6 +24,14 @@ void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * G + void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k); + void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k); + void quantize_row_turbo8_0_ref(const float * GGML_RESTRICT x, block_turbo8_0 * GGML_RESTRICT y, int64_t k); ++// opencoti-hook: TCQ (F5 M6 C5) — CPU from_float refs (norm-only stubs) must also be declared here: ++// ggml-cpu.c includes this CPU-local quants.h, NOT the global ggml-quants.h, for the from_float table. ++void quantize_row_turbo3_tcq_ref(const float * GGML_RESTRICT x, block_turbo3_tcq * GGML_RESTRICT y, int64_t k); ++void quantize_row_turbo2_tcq_ref(const float * GGML_RESTRICT x, block_turbo2_tcq * GGML_RESTRICT y, int64_t k); ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — same CPU-local-header gotcha: ggml-cpu.c's q6_0 ++// from_float = quantize_row_q6_0_ref resolves against THIS header, not the global ggml-quants.h, so the ++// q6_0 ref must be declared here too (defined in ggml-quants.c). ++void quantize_row_q6_0_ref(const float * GGML_RESTRICT x, block_q6_0 * GGML_RESTRICT y, int64_t k); + + void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +diff --git a/llama.cpp/ggml/src/ggml-cuda/common.cuh b/llama.cpp/ggml/src/ggml-cuda/common.cuh +index c9275517f..1fdf9d02a 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh +@@ -21,6 +21,19 @@ + #endif + #include "ggml-common.h" + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). The host ggml-common.h defines QK6_0 (32) but ++// not the QR/QI dequant-reduction macros (q5_0's QR5_0/QI5_0 live there; q6_0's were never added ++// host-side, and host files are frozen for this port). Define them CUDA-side here, right after ++// ggml-common.h (where QK6_0 lands), so convert.cu's dequantize_block_cont_cuda and ++// fattn-common.cuh's K reader (QI6_0) resolve. q6_0 packs 2 values per qs byte ⇒ QR6_0 = 2 (mirrors ++// QR5_0); QI6_0 = QK6_0/(4*QR6_0) = 4 (int32 lanes of qs per block, used by the FA-VEC iqs index). ++#ifndef QR6_0 ++#define QR6_0 2 ++#endif ++#ifndef QI6_0 ++#define QI6_0 (QK6_0 / (4 * QR6_0)) ++#endif ++ + #include + #include + #include +diff --git a/llama.cpp/ggml/src/ggml-cuda/convert.cu b/llama.cpp/ggml/src/ggml-cuda/convert.cu +index 0d27c1672..6ee9aa92b 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu +@@ -720,6 +720,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { + return dequantize_row_q4_1_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cont_cuda; ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_block_cont_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0: +@@ -789,6 +791,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { + return dequantize_row_q4_1_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cont_cuda; ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_block_cont_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0: +@@ -848,6 +852,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { + return dequantize_block_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cuda; ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_block_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0: +@@ -880,6 +886,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { + return dequantize_block_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cuda; ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_block_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0: +@@ -903,6 +911,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { + return dequantize_block_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cuda; ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_block_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cuda; + case GGML_TYPE_Q8_0: +diff --git a/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh b/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh +index 7697c292d..d82afeb18 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh +@@ -103,6 +103,43 @@ static __device__ void quantize_f32_q5_0_block(const float * __restrict__ x, blo + memcpy(y->qh, &qh, sizeof(qh)); + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). f32->q6_0 block pack, modeled on ++// quantize_f32_q5_0_block above (our amax/vmax scan + per-half loop idiom). Pack math ported verbatim ++// from ik_llama.cpp cpy-utils.cuh:210-241 (quantize_f32_q6_0_block): d = vmax/-32 (NO sumqx/sumq2 refit ++// for q6_0 — ik only refits iq4_nl); 6-bit codes via min(63, (int8_t)(x*id + 32.5)); qs holds low ++// nibbles, qh[8] holds the high 2 bits packed 4 values/byte (h = (xi0>>4)|((xi1>>4)<<2), then ++// qh[j%8] |= h << 4*(j/8)). Produces bytes byte-identical to the host quantize_row_q6_0_ref. ++static __device__ void quantize_f32_q6_0_block(const float * __restrict__ x, block_q6_0 * __restrict__ y) { ++ float amax = 0.0f; ++ float vmax = 0.0f; ++ ++ for (int j = 0; j < QK6_0; ++j) { ++ const float v = x[j]; ++ if (amax < fabsf(v)) { ++ amax = fabsf(v); ++ vmax = v; ++ } ++ } ++ ++ const float d = vmax / -32; ++ const float id = d ? 1.0f/d : 0.0f; ++ ++ y->d = d; ++ memset(y->qh, 0, QK6_0/4); ++ ++ for (int j = 0; j < QK6_0/2; ++j) { ++ const float x0 = x[0 + j]*id; ++ const float x1 = x[QK6_0/2 + j]*id; ++ ++ const uint8_t xi0 = min(63, (int8_t)(x0 + 32.5f)); ++ const uint8_t xi1 = min(63, (int8_t)(x1 + 32.5f)); ++ ++ y->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); ++ const uint8_t h = (xi0 >> 4) | ((xi1 >> 4) << 2); ++ y->qh[j % (QK6_0/4)] |= (h << 4*(j / (QK6_0/4))); ++ } ++} ++ + static __device__ void quantize_f32_q5_1_block(const float * __restrict__ x, block_q5_1 * __restrict__ y) { + float min = x[0]; + float max = x[0]; +@@ -203,6 +240,11 @@ static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) { + quantize_f32_q5_1_block((const float *)cxi, (block_q5_1 *)cdsti); + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). cpy.cu wrapper, mirrors cpy_blck_f32_q5_0. ++static __device__ void cpy_blck_f32_q6_0(const char * cxi, char * cdsti) { ++ quantize_f32_q6_0_block((const float *)cxi, (block_q6_0 *)cdsti); ++} ++ + static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { + quantize_f32_q8_0_block((const float *)cxi, (block_q8_0 *)cdsti); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/cpy.cu b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +index 5e01613fd..e769b7bf7 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +@@ -344,6 +344,35 @@ static void ggml_cpy_q5_0_f32_cuda( + ne10, ne11, ne12, nb10, nb11, nb12, nb13); + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). f32->q6_0 / q6_0->f32 cpy launchers, mirror ++// ggml_cpy_f32_q5_0_cuda / ggml_cpy_q5_0_f32_cuda above with QK6_0 block size and the q6_0 ++// pack/dequant block functions (cpy_blck_f32_q6_0 / dequantize_q6_0). ++static void ggml_cpy_f32_q6_0_cuda( ++ const char * cx, char * cdst, const int64_t ne, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, ++ const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { ++ ++ GGML_ASSERT(ne % QK6_0 == 0); ++ const int64_t num_blocks = ne / QK6_0; ++ GGML_ASSERT(num_blocks < UINT_MAX); ++ cpy_f32_q<<>> ++ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); ++} ++ ++static void ggml_cpy_q6_0_f32_cuda( ++ const char * cx, char * cdst, const int64_t ne, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t nb00, const int64_t nb01, const int64_t nb02, ++ const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, ++ const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, ++ cudaStream_t stream) { ++ const int64_t num_blocks = ne; ++ GGML_ASSERT(num_blocks < UINT_MAX); ++ cpy_q_f32, QK6_0><<>>( ++ cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ++ ne10, ne11, ne12, nb10, nb11, nb12, nb13); ++} ++ + static void ggml_cpy_f32_q5_1_cuda( + const char * cx, char * cdst, const int64_t ne, + const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, +@@ -477,6 +506,13 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg + } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q5_0_f32_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — f32<->q6_0 cpy dispatch, mirrors q5_0. ++ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q6_0) { ++ ggml_cpy_f32_q6_0_cuda ++ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); ++ } else if (src0->type == GGML_TYPE_Q6_0 && src1->type == GGML_TYPE_F32) { ++ ggml_cpy_q6_0_f32_cuda ++ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + #ifndef GGML_CUDA_NO_IQ_QUANTS + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { + ggml_cpy_f32_iq4_nl_cuda +diff --git a/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh b/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh +index 9ae1342fc..8e022111e 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh +@@ -68,6 +68,25 @@ static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const in + v.y = (v.y - 16.0f) * d; + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Scalar block dequantizer (2 values/call), ++// modeled on dequantize_q5_0 above (our float2& idiom; our tree has no GGML_CUDA_F16 branch here). ++// Bit-unpack ported verbatim from ik_llama.cpp dequantize.cuh:107-123 (dequantize_q6_0): ++// h = qh[iqs%8] >> 4*(iqs/8); value iqs = (qs[iqs]&0xf) | ((h&0x3)<<4); value iqs+16 = ++// (qs[iqs]>>4) | ((h&0xc)<<2); both minus the q6_0 level offset 32. Used by convert.cu (to_fpXX) ++// and cpy.cu (q6_0->f32) via dequantize_block_cont_cuda. ++static __device__ __forceinline__ void dequantize_q6_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ ++ const block_q6_0 * x = (const block_q6_0 *) vx; ++ ++ const float d = x[ib].d; ++ ++ const uint8_t h = x[ib].qh[iqs%8] >> 4*(iqs/8); ++ v.x = ((x[ib].qs[iqs] & 0xf) | ((h & 0x3) << 4)); ++ v.y = ((x[ib].qs[iqs] >> 4) | ((h & 0xc) << 2)); ++ ++ v.x = (v.x - 32.0f) * d; ++ v.y = (v.y - 32.0f) * d; ++} ++ + static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_q5_1 * x = (const block_q5_1 *) vx; + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +index f61cd9d37..e5181d703 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -232,6 +232,54 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q5_0( + return sum; + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). K reader modeled on vec_dot_fattn_vec_KQ_q5_0 ++// above (our float-return idiom), with the q6_0 bit-unpack math ported verbatim from ++// ik_llama.cpp fattn-common.cuh:290-331 (vec_dot_fattn_vec_KQ_q6_0). Differences from q5_0: q6_0 packs ++// the high 2 bits across qh[8] (4 values/byte) rather than 1 bit/value in a uint32 qh, the level ++// offset is 32 (not 16), and the Q_ds combine uses (32/QI8_1) instead of (16/QI8_1). q6_0 carries no ++// min term (symmetric, like q5_0) so the float path mirrors q5_0's single sumi*Q_ds.x - off*Q_ds.y. ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q6_0( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ ++ const block_q6_0 * K_q6_0 = (const block_q6_0 *) K_c; ++ GGML_UNUSED(Q_v); ++ ++ float sum = 0.0f; ++ ++#pragma unroll ++ for (int k_KQ_0 = 0; k_KQ_0 < int(D/sizeof(int)); k_KQ_0 += nthreads) { ++ const int k_KQ = k_KQ_0 + (nthreads == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads); ++ ++ const int ib = k_KQ / QI8_1; ++ const int iqs4 = k_KQ % QI6_0; // 0...3 ++ const int shift = k_KQ & (QI8_1/2); ++ ++ // High 2 bits packed across qh[8] (4 values/byte); ik selects the qh int by iqs4%2 and shifts ++ // by 4*(iqs4/2) + shift/2 to land on this lane's 2-bit field, masked to 0x03 per byte. ++ int vh; ++ ggml_cuda_memcpy_1(&vh, K_q6_0[ib].qh + sizeof(int)*(iqs4 % 2)); ++ vh = (vh >> (4*(iqs4/2) + shift/2)) & 0x03030303; ++ ++ // Low 4 bits: one int (4 values) of qs at lane iqs4, shifted by `shift` (0 or 4), masked to nibble. ++ int vl; ++ ggml_cuda_memcpy_1(&vl, K_q6_0[ib].qs + sizeof(int)*iqs4); ++ vl = (vl >> shift) & 0x0F0F0F0F; ++ ++ const int v = vl | (vh << 4); ++ ++ const int u = Q_q8[k_KQ_0/nthreads]; ++ ++ const int sumi = ggml_cuda_dp4a(v, u, 0); ++ ++ const float2 Q_ds = ((const float2 *) Q_ds_v)[k_KQ_0/nthreads]; ++ ++ sum += __half2float(K_q6_0[ib].d) * (sumi*Q_ds.x - (32/QI8_1)*Q_ds.y); ++ } ++ ++ return sum; ++} ++ + template + static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q5_1( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { +@@ -512,6 +560,66 @@ static __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict + } + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). V reader modeled on dequantize_V_q5_0 above ++// (our block-fill idiom: read ne low-nibbles into a packed int, OR in the high bits per lane, ++// subtract the level offset, scale by d). The high-bit unpack + offset(32) math is ported from ++// ik_llama.cpp fattn-common.cuh:566-588 (dequantize_1_q6_0), generalized to ne lanes: for output lane ++// l at absolute index idq+l, the high 2 bits come from qh[(idq+l)%8] >> 4*((idq+l)/8), then &0x03<<4. ++template ++static __device__ __forceinline__ void dequantize_V_q6_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ const block_q6_0 * x = (const block_q6_0 *) vx; ++ ++ const int64_t ib = i0 / QK6_0; ++ const int idq = i0 % QK6_0; ++ const int iqs = i0 % (QK6_0/2); ++ const int shift = (i0 % QK6_0) / (QK6_0/2); ++ ++ int q; ++ static_assert(ne == 2 || ne == 4, "bad ne"); ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; ++ ++ { ++ // High 2 bits, ported verbatim from ik_llama.cpp dequantize_1_q6_0 (fattn-common.cuh:577-578) ++ // and evaluated per output lane l at value index (idq + l). For value V: byte = qh[V%(QK6_0/4)], ++ // 2-bit field at shift = 4*((V/(QK6_0/4))%2) + 2*shift. `shift` is constant across the ne lanes ++ // of one call (i0 is aligned within a nibble-half), so it equals (idq+l)/(QK6_0/2) for every l. ++ // Verified byte-exact against the host quantize_row_q6_0 pack for all 32 block positions. ++#pragma unroll ++ for (int l = 0; l < ne; ++l) { ++ const int j = idq + l; ++ const int h = (int) x[ib].qh[j % (QK6_0/4)] >> (4*((j / (QK6_0/4)) % 2) + 2*shift); ++ q |= (h & 0x03) << (8*l + 4); ++ } ++ } ++ ++ q = __vsubss4(q, 0x20202020); // q6_0 level offset is 32 (0x20), vs q5_0's 16 ++ ++ const int8_t * q8 = (const int8_t *) &q; ++ ++#ifdef FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ const half2 d = __half2half2(x[ib].d); ++ ++#pragma unroll ++ for (int l0 = 0; l0 < ne; l0 += 2) { ++ ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ } ++ } else ++#endif // FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ const float d = x[ib].d; ++ ++#pragma unroll ++ for (int l = 0; l < ne; ++l) { ++ ((float *) dst)[l] = d * q8[l]; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); ++ } ++} ++ + template + static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q5_1 * x = (const block_q5_1 *) vx; +@@ -599,6 +707,10 @@ static __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict + // (centroid×norm, rotated space) for the fused FA-VEC path. Included here so get_vec_dot_KQ / + // get_dequantize_V below can dispatch turbo2/3/4. ggml_cuda_mad + block_turbo* are already in scope. + #include "fattn-turbo.cuh" ++// opencoti-hook: TCQ (trellis-coded quant) FA-VEC decode — F5 M6 C5 (#447/#500). Decode codebooks + ++// O(1) sliding-window trellis readers, ported verbatim from buun. Included after fattn-turbo.cuh so ++// block_turbo{2,3}_tcq / QK_TURBO{2,3}_TCQ / cpy-bytes helper are in scope. See UPSTREAM_SYNC.md. ++#include "fattn-tcq.cuh" + + template + constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() { +@@ -610,12 +722,18 @@ constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() { + return vec_dot_fattn_vec_KQ_turbo3; + } else if constexpr (type_K == GGML_TYPE_TURBO4_0) { + return vec_dot_fattn_vec_KQ_turbo4; ++ } else if constexpr (type_K == GGML_TYPE_TURBO3_TCQ) { // opencoti-hook: TCQ FA-VEC decode (#447/#500) ++ return vec_dot_fattn_vec_KQ_turbo3_tcq; ++ } else if constexpr (type_K == GGML_TYPE_TURBO2_TCQ) { ++ return vec_dot_fattn_vec_KQ_turbo2_tcq; + } else if constexpr (type_K == GGML_TYPE_Q4_0) { + return vec_dot_fattn_vec_KQ_q4_0; + } else if constexpr (type_K == GGML_TYPE_Q4_1) { + return vec_dot_fattn_vec_KQ_q4_1; + } else if constexpr (type_K == GGML_TYPE_Q5_0) { + return vec_dot_fattn_vec_KQ_q5_0; ++ } else if constexpr (type_K == GGML_TYPE_Q6_0) { // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return vec_dot_fattn_vec_KQ_q6_0; + } else if constexpr (type_K == GGML_TYPE_Q5_1) { + return vec_dot_fattn_vec_KQ_q5_1; + } else if constexpr (type_K == GGML_TYPE_Q8_0) { +@@ -638,12 +756,18 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { + return dequantize_V_turbo3; + } else if constexpr (type_V == GGML_TYPE_TURBO4_0) { + return dequantize_V_turbo4; ++ } else if constexpr (type_V == GGML_TYPE_TURBO3_TCQ) { // opencoti-hook: TCQ FA-VEC decode (#447/#500) ++ return dequantize_V_turbo3_tcq; ++ } else if constexpr (type_V == GGML_TYPE_TURBO2_TCQ) { ++ return dequantize_V_turbo2_tcq; + } else if constexpr (type_V == GGML_TYPE_Q4_0) { + return dequantize_V_q4_0; + } else if constexpr (type_V == GGML_TYPE_Q4_1) { + return dequantize_V_q4_1; + } else if constexpr (type_V == GGML_TYPE_Q5_0) { + return dequantize_V_q5_0; ++ } else if constexpr (type_V == GGML_TYPE_Q6_0) { // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++ return dequantize_V_q6_0; + } else if constexpr (type_V == GGML_TYPE_Q5_1) { + return dequantize_V_q5_1; + } else if constexpr (type_V == GGML_TYPE_Q8_0) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-tcq.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-tcq.cuh +new file mode 100644 +index 000000000..a9541f575 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-tcq.cuh +@@ -0,0 +1,304 @@ ++#pragma once ++// opencoti-hook: TCQ (trellis-coded quant) FA-VEC decode — F5 M6 C5 (#447/#500). See docs/protocols/UPSTREAM_SYNC.md. ++// Ported VERBATIM from buun-llama-cpp (github.com/spiritbuun/buun-llama-cpp @87c351d) fattn-common.cuh: ++// decode codebooks (cross-validated byte-identical to the encode codebooks in turbo-tcq-cuda.cuh), ++// context-adaptive decode alphas, and the O(1) sliding-window trellis readers (9-bit/512-state for ++// turbo3, 8-bit/256-state for turbo2). buun naming retained per project decision. ++// Included by fattn-common.cuh AFTER fattn-turbo.cuh, so these are already in scope: ++// block_turbo{2,3}_tcq, QK_TURBO{2,3}_TCQ (=128), ggml_cuda_get_max_cpy_bytes(), ++// V_DOT2_F32_F16_AVAILABLE, FP16_AVAILABLE, GGML_UNUSED. ++ ++// --- 3-bit TCQ decode codebook (512-state), verbatim from buun fattn-common.cuh:60-125 --- ++static __constant__ float d_turbo3_tcq_codebook_fattn[512] = { ++ -0.14559399f, -0.09062801f, -0.054925077f, -0.03699251f, -0.006363985f, +0.026264573f, +0.067378916f, +0.121981815f, ++ -0.18648055f, -0.106522456f, -0.052047577f, -0.011695214f, +0.021953275f, +0.059698727f, +0.09831437f, +0.16083933f, ++ -0.16390342f, -0.12639847f, -0.09513180f, -0.05938352f, -0.028396897f, +0.005973862f, +0.049104784f, +0.11334257f, ++ -0.25952467f, -0.079778515f, -0.036024813f, +0.0003641268f, +0.031858794f, +0.073280424f, +0.11835553f, +0.19738495f, ++ -0.14218009f, -0.10224814f, -0.062498566f, -0.027066832f, +0.00393002f, +0.04069300f, +0.08257346f, +0.14548601f, ++ -0.18673635f, -0.13438253f, -0.088401966f, -0.05205436f, -0.02032501f, +0.012399545f, +0.05127183f, +0.10316186f, ++ -0.10807011f, -0.065903045f, -0.032206114f, -0.0062006037f, +0.020679146f, +0.04422085f, +0.08313074f, +0.16821936f, ++ -0.22979105f, -0.14431947f, -0.07689272f, -0.02755307f, +0.009225173f, +0.046684854f, +0.08834142f, +0.13766693f, ++ -0.22114082f, -0.12612148f, -0.06890522f, -0.016128855f, +0.03691900f, +0.08474852f, +0.14940020f, +0.23229980f, ++ -0.14933491f, -0.099693604f, -0.06738499f, -0.037100967f, -0.009332986f, +0.023535024f, +0.060272533f, +0.109464675f, ++ -0.20200425f, -0.07398328f, -0.038700905f, -0.01714807f, +0.011161969f, +0.04528101f, +0.08902637f, +0.19573534f, ++ -0.16645233f, -0.124482535f, -0.089342155f, -0.04427387f, -0.007353691f, +0.028033108f, +0.066108435f, +0.15552913f, ++ -0.22295763f, -0.059887577f, -0.018804537f, +0.020141022f, +0.059682943f, +0.097920544f, +0.14080113f, +0.25698325f, ++ -0.14248224f, -0.089685425f, -0.050101686f, -0.017257255f, +0.011412255f, +0.040830314f, +0.07400172f, +0.11997315f, ++ -0.18649384f, -0.113997504f, -0.067775466f, -0.033394672f, +0.006586988f, +0.05312057f, +0.10433043f, +0.22344802f, ++ -0.16138338f, -0.108194515f, -0.07600300f, -0.05135381f, -0.023365447f, +0.0087320795f, +0.045431953f, +0.09113002f, ++ -0.12630440f, -0.07225349f, -0.032280035f, +0.0029231994f, +0.019239848f, +0.05081419f, +0.077840395f, +0.121695265f, ++ -0.08928155f, -0.044983763f, -0.009889568f, +0.020831043f, +0.05684458f, +0.09409702f, +0.13867535f, +0.19084482f, ++ -0.14182915f, -0.11380146f, -0.06904074f, -0.002002765f, +0.034864165f, +0.070399575f, +0.11403063f, +0.15394832f, ++ -0.10876417f, -0.056122433f, -0.02267638f, +0.011113975f, +0.039639056f, +0.074084364f, +0.10155376f, +0.12540291f, ++ -0.17693359f, -0.13940524f, -0.10049578f, -0.06796275f, -0.036915872f, +0.00062823476f, +0.042142134f, +0.17906062f, ++ -0.09253492f, -0.04290128f, -0.006311852f, +0.023908244f, +0.049849935f, +0.078770354f, +0.10818172f, +0.15166481f, ++ -0.12429565f, -0.07392063f, -0.029114135f, +0.0059440783f, +0.042675965f, +0.08425635f, +0.13836108f, +0.18634140f, ++ -0.11795639f, -0.07033707f, -0.034163877f, -0.0008773357f, +0.03334606f, +0.07188203f, +0.12216825f, +0.17097956f, ++ -0.18718453f, -0.14090346f, -0.097799584f, -0.059522875f, -0.019208657f, +0.03079176f, +0.09334672f, +0.15811224f, ++ -0.27198875f, -0.16546582f, -0.11433405f, -0.06933013f, -0.04026183f, -0.0061146915f, +0.029263576f, +0.07322499f, ++ -0.18471734f, -0.102074504f, -0.06492570f, -0.034418534f, -0.009636157f, +0.023043344f, +0.05751496f, +0.09905984f, ++ -0.22826399f, -0.15946552f, -0.09913176f, -0.06585259f, -0.03252090f, +0.001313243f, +0.03556729f, +0.21612854f, ++ -0.13243781f, -0.087299444f, -0.049820945f, -0.016216082f, +0.01799807f, +0.057916876f, +0.09001349f, +0.13221787f, ++ -0.19516511f, -0.120894566f, -0.076130204f, -0.051442243f, -0.029535033f, -0.0020043184f, +0.029452588f, +0.075566076f, ++ -0.27272871f, -0.15841717f, -0.105432935f, -0.06792948f, -0.024532158f, +0.014960791f, +0.054415092f, +0.101517834f, ++ -0.21153601f, -0.15015371f, -0.08676790f, -0.04414934f, -0.0042129597f, +0.033762872f, +0.07589151f, +0.12768789f, ++ -0.090428725f, -0.037582967f, +0.0013173596f, +0.03900247f, +0.06840049f, +0.116906695f, +0.16584939f, +0.25382105f, ++ -0.13446195f, -0.07865091f, -0.039625354f, -0.0028398742f, +0.03019514f, +0.06799379f, +0.11850997f, +0.17521496f, ++ -0.11350345f, -0.058599845f, -0.017512511f, +0.019431496f, +0.055897832f, +0.093173414f, +0.14820710f, +0.22092152f, ++ -0.15165758f, -0.08869354f, -0.04974287f, -0.01705474f, +0.013134752f, +0.04367713f, +0.07733791f, +0.12430801f, ++ -0.09329869f, -0.04673005f, -0.00045857552f, +0.042781368f, +0.07802363f, +0.11887439f, +0.16250038f, +0.28612965f, ++ -0.12571070f, -0.07786012f, -0.03843933f, -0.0075433915f, +0.025822964f, +0.066053316f, +0.12021536f, +0.18341768f, ++ -0.16079275f, -0.04921760f, -0.006114644f, +0.026215268f, +0.05699377f, +0.09813471f, +0.16080129f, +0.23786584f, ++ -0.09980837f, -0.048535258f, -0.0096120685f, +0.025387142f, +0.05979822f, +0.09875251f, +0.14474337f, +0.20324114f, ++ -0.15846540f, -0.09938028f, -0.061492465f, -0.03523542f, -0.0061364113f, +0.024916094f, +0.06037314f, +0.106796466f, ++ -0.20557843f, -0.123237535f, -0.07734871f, -0.044549115f, -0.017114898f, +0.01616654f, +0.049574375f, +0.092319444f, ++ -0.19221115f, -0.14642999f, -0.091701314f, -0.055265956f, -0.021026207f, +0.017720066f, +0.05786183f, +0.110154524f, ++ -0.09956386f, -0.03870283f, +0.003052007f, +0.034851722f, +0.06256365f, +0.09628840f, +0.13979156f, +0.16582295f, ++ -0.18026546f, -0.12448310f, -0.07424377f, -0.03954519f, -0.01221123f, +0.028641058f, +0.100819774f, +0.18240699f, ++ -0.21520759f, -0.15573645f, -0.09820838f, -0.051450998f, -0.012993679f, +0.021135861f, +0.058727216f, +0.105848536f, ++ -0.11207385f, -0.08335689f, -0.048542723f, -0.023198519f, +0.0039304253f, +0.037778318f, +0.07813917f, +0.13106476f, ++ -0.17849164f, -0.120988995f, -0.078016765f, -0.043093704f, -0.016565649f, +0.015182641f, +0.050754096f, +0.09595712f, ++ -0.22132620f, -0.13407415f, -0.065785654f, -0.013291034f, +0.032098345f, +0.07478225f, +0.12431934f, +0.19174045f, ++ -0.095454164f, -0.051898945f, -0.015116375f, -0.012596778f, +0.018636847f, +0.05006925f, +0.087654814f, +0.13754296f, ++ -0.15254061f, -0.09576059f, -0.052086458f, -0.01596074f, +0.017607626f, +0.04778498f, +0.08950204f, +0.14901252f, ++ -0.26057002f, -0.12472382f, -0.074396215f, -0.03764066f, +0.0011168446f, +0.061569117f, +0.10793752f, +0.19771695f, ++ -0.08661132f, -0.045195263f, -0.016098704f, +0.012780116f, +0.040476497f, +0.074102715f, +0.074102715f, +0.12635531f, ++ -0.14047913f, -0.059587404f, -0.016261123f, +0.019801628f, +0.053541403f, +0.096650146f, +0.15005490f, +0.21051759f, ++ -0.22986396f, -0.11964334f, -0.07266585f, -0.026522418f, +0.018169926f, +0.058630653f, +0.100647695f, +0.15919648f, ++ -0.13251697f, -0.077567816f, -0.042766172f, -0.011389967f, +0.01831755f, +0.05304656f, +0.09620367f, +0.15567583f, ++ -0.119819686f, -0.06772876f, -0.028123451f, +0.00876240f, +0.014405836f, +0.048829112f, +0.08422175f, +0.13823749f, ++ -0.16379014f, -0.08956941f, -0.041652776f, +0.008921398f, +0.05473602f, +0.10037984f, +0.16022855f, +0.23457925f, ++ -0.115844205f, -0.05939626f, -0.020390417f, +0.01374377f, +0.044976473f, +0.07873563f, +0.12207942f, +0.18412720f, ++ -0.19048831f, -0.07587487f, -0.03220580f, -0.00011795067f, +0.02721784f, +0.04380719f, +0.07886723f, +0.13193911f, ++ -0.13935551f, -0.092902906f, -0.052706074f, -0.017797327f, +0.015312965f, +0.056098964f, +0.11203423f, +0.24448302f, ++ -0.17986591f, -0.10738580f, -0.06376371f, -0.026595421f, +0.00842492f, +0.04272362f, +0.08608052f, +0.15240218f, ++ -0.10953678f, -0.057022586f, -0.012483291f, +0.024463262f, +0.06076792f, +0.09776234f, +0.12983681f, +0.18648379f, ++ -0.16471463f, -0.089491285f, -0.037574016f, +0.004444791f, +0.039293647f, +0.07845859f, +0.12893885f, +0.23508036f ++}; ++ ++// --- 2-bit TCQ decode codebook (256-state), verbatim from buun fattn-common.cuh:129-162 --- ++static __constant__ float d_turbo2_tcq_codebook_fattn[256] = { ++ -0.18030643f, -0.11009848f, -0.04742626f, +0.02894132f, -0.10523465f, -0.031312924f, +0.031491395f, +0.12263535f, ++ -0.15660362f, -0.055477407f, +0.0046675834f, +0.06166081f, -0.07506216f, -0.016963918f, +0.043737844f, +0.116496615f, ++ -0.08632783f, -0.022493735f, +0.041032985f, +0.10660284f, -0.06274858f, -0.0036939639f, +0.02095157f, +0.07539709f, ++ -0.09802641f, -0.008419088f, +0.059072323f, +0.17311879f, -0.093109086f, -0.02654333f, +0.014827672f, +0.07793592f, ++ -0.031235758f, +0.01271591f, +0.08752262f, +0.17246453f, -0.14595252f, -0.07227624f, +0.013628688f, +0.08131674f, ++ -0.036909282f, +0.0018896917f, +0.05209119f, +0.12407892f, -0.13689458f, -0.06054520f, +0.0064648795f, +0.07551241f, ++ -0.18980840f, -0.110128626f, -0.046503957f, +0.026387159f, -0.034967307f, +0.04810357f, +0.072072044f, +0.14355458f, ++ -0.10182410f, -0.02907887f, +0.014033012f, +0.083419636f, -0.056140676f, +0.008405868f, +0.066070884f, +0.14037225f, ++ -0.117427245f, -0.047159385f, +0.016928354f, +0.08142885f, -0.029359628f, +0.045608785f, +0.10559447f, +0.20061271f, ++ -0.040425077f, +0.029068163f, +0.08408973f, +0.13628258f, -0.16633821f, -0.10711727f, -0.04196669f, +0.027895834f, ++ -0.0054065837f, +0.058898676f, +0.12688550f, +0.18268861f, -0.16287325f, -0.11218357f, -0.07165227f, -0.009524379f, ++ -0.24026902f, -0.073219374f, -0.0005165726f, +0.05959821f, -0.05532953f, +0.027044486f, +0.09425678f, +0.15356481f, ++ -0.14381111f, -0.10563502f, -0.037867088f, +0.023611993f, -0.03624307f, +0.049588434f, +0.12192037f, +0.23462485f, ++ -0.14990251f, -0.09659304f, -0.05886742f, +0.014878461f, -0.009889551f, +0.06910514f, +0.12120181f, +0.22596690f, ++ -0.08290075f, -0.009009629f, +0.066151775f, +0.12188313f, -0.11591514f, -0.06952189f, -0.031633306f, +0.023740824f, ++ -0.20510401f, -0.103369795f, +0.09148037f, +0.17268716f, -0.16597997f, -0.09207068f, -0.032810967f, +0.024847647f, ++ -0.02487482f, +0.049298953f, +0.09624215f, +0.14217524f, -0.18418685f, -0.10147012f, -0.05841265f, +0.008057022f, ++ -0.14269894f, -0.092456274f, -0.026881337f, +0.049792137f, -0.019881032f, +0.030333601f, +0.09736802f, +0.17764080f, ++ -0.19579841f, -0.114739306f, -0.026823774f, +0.07466014f, -0.09001050f, -0.041468445f, +0.028473806f, +0.08870695f, ++ -0.019396419f, +0.042828932f, +0.10885327f, +0.13335012f, -0.15005013f, -0.074581385f, -0.028608415f, +0.03848942f, ++ -0.09687270f, -0.057059396f, +0.0077843578f, +0.06302297f, -0.23247094f, -0.14509225f, -0.032651436f, +0.027010715f, ++ -0.047595482f, +0.06280303f, +0.114691675f, +0.17124057f, -0.21092793f, -0.13704823f, -0.07340412f, +0.0039013291f, ++ -0.062834196f, +0.012601906f, +0.012601906f, +0.08721347f, -0.13256435f, -0.024173854f, +0.07723171f, +0.14801070f, ++ -0.06471605f, -0.0017903054f, -0.0017903054f, +0.058302354f, -0.09731802f, -0.03400696f, +0.02762442f, +0.08986137f, ++ -0.08288722f, -0.019051429f, +0.045709886f, +0.15211061f, -0.09507891f, -0.015612489f, +0.025347246f, +0.087257534f, ++ -0.066236064f, -0.0047936034f, +0.06386274f, +0.15401669f, -0.105809286f, -0.051802177f, +0.01073050f, +0.08292137f, ++ -0.11884470f, -0.04404144f, +0.02550729f, +0.02550729f, -0.01731189f, +0.062161792f, +0.12127554f, +0.21981733f, ++ -0.17066145f, -0.11660990f, -0.049425896f, +0.021293938f, -0.04711412f, +0.026577346f, +0.055197213f, +0.12541275f, ++ -0.028268812f, +0.015206398f, +0.09002519f, +0.12699963f, -0.10059831f, -0.026676945f, +0.059903253f, +0.13054545f, ++ -0.09582803f, -0.033371232f, +0.010346129f, +0.066766635f, -0.09964944f, -0.028686784f, +0.021184925f, +0.09120017f, ++ -0.16957201f, -0.07594450f, +0.04172865f, +0.18313301f, -0.051526368f, +0.011877304f, +0.011877304f, +0.07956263f, ++ -0.13432936f, -0.05269006f, +0.03536416f, +0.117640756f, -0.022776067f, +0.042032316f, +0.10472976f, +0.18042557f ++}; ++ ++// TCQ decode-time alphas: per-compilation-unit copies, set by cudaMemcpyToSymbol before kernel launch ++static __constant__ float d_tcq_decode_alpha_k_fattn = 1.0f; ++static __constant__ float d_tcq_decode_alpha_v_fattn = 1.0f; ++ ++// TCQ 3-bit K dot product: 9-bit state → codebook lookup ++// Core implementation takes explicit codebook pointer for SMEM/constant flexibility ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3_tcq_cb( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, ++ const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, ++ const float * __restrict__ cb) { ++ const block_turbo3_tcq * K_tcq = (const block_turbo3_tcq *) K_c; ++ GGML_UNUSED(Q_q8); GGML_UNUSED(Q_ds_v); ++ constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes(); ++ constexpr int cpy_ne = cpy_nb / 4; ++ float sum = 0.0f; ++ int prev_ib = -1; ++ float norm = 0.0f; ++#pragma unroll ++ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) { ++ const int base_f2 = k_KQ_0 + (threadIdx.x % nthreads) * cpy_ne; ++ const int elem0 = base_f2 * 2; ++ const int ib = elem0 / QK_TURBO3_TCQ; ++ const int j_start = elem0 % QK_TURBO3_TCQ; ++ ++ if (ib != prev_ib) { ++ norm = __half2float(K_tcq[ib].norm); ++ prev_ib = ib; ++ } ++ ++#pragma unroll ++ for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) { ++ const int lj = k_KQ_1 * 2; ++ const int t0 = j_start + lj; ++ const int t1 = t0 + 1; ++ const int bp0 = t0 * 3; ++ const uint16_t raw0 = (uint16_t)K_tcq[ib].qs[bp0/8] | ((uint16_t)K_tcq[ib].qs[bp0/8 + 1] << 8); ++ const float k0 = cb[(raw0 >> (bp0 % 8)) & 0x1FF] * norm; ++ const int bp1 = t1 * 3; ++ const uint16_t raw1 = (uint16_t)K_tcq[ib].qs[bp1/8] | ((uint16_t)K_tcq[ib].qs[bp1/8 + 1] << 8); ++ const float k1 = cb[(raw1 >> (bp1 % 8)) & 0x1FF] * norm; ++#ifdef V_DOT2_F32_F16_AVAILABLE ++ const float2 qf = __half22float2(((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); ++#else ++ const float2 qf = ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]; ++#endif ++ sum += k0 * qf.x + k1 * qf.y; ++ } ++ } ++ return sum; ++} ++ ++// Wrapper using __constant__ codebook (for function pointer dispatch) ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3_tcq( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, ++ const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ return vec_dot_fattn_vec_KQ_turbo3_tcq_cb(K_c, Q_v, Q_q8, Q_ds_v, d_turbo3_tcq_codebook_fattn); ++} ++ ++// TCQ 2-bit K dot product: 8-bit state → codebook lookup ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2_tcq_cb( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, ++ const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, ++ const float * __restrict__ cb) { ++ const block_turbo2_tcq * K_tcq = (const block_turbo2_tcq *) K_c; ++ GGML_UNUSED(Q_q8); GGML_UNUSED(Q_ds_v); ++ constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes(); ++ constexpr int cpy_ne = cpy_nb / 4; ++ float sum = 0.0f; ++ int prev_ib = -1; ++ float norm = 0.0f; ++#pragma unroll ++ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) { ++ const int base_f2 = k_KQ_0 + (threadIdx.x % nthreads) * cpy_ne; ++ const int elem0 = base_f2 * 2; ++ const int ib = elem0 / QK_TURBO2_TCQ; ++ const int j_start = elem0 % QK_TURBO2_TCQ; ++ ++ if (ib != prev_ib) { ++ norm = __half2float(K_tcq[ib].norm); ++ prev_ib = ib; ++ } ++ ++#pragma unroll ++ for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) { ++ const int lj = k_KQ_1 * 2; ++ const int t0 = j_start + lj; ++ const int t1 = t0 + 1; ++ const int bp0 = t0 * 2; ++ const uint16_t raw0 = (uint16_t)K_tcq[ib].qs[bp0/8] | ((uint16_t)K_tcq[ib].qs[bp0/8 + 1] << 8); ++ const float k0 = cb[(raw0 >> (bp0 % 8)) & 0xFF] * norm; ++ const int bp1 = t1 * 2; ++ const uint16_t raw1 = (uint16_t)K_tcq[ib].qs[bp1/8] | ((uint16_t)K_tcq[ib].qs[bp1/8 + 1] << 8); ++ const float k1 = cb[(raw1 >> (bp1 % 8)) & 0xFF] * norm; ++#ifdef V_DOT2_F32_F16_AVAILABLE ++ const float2 qf = __half22float2(((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); ++#else ++ const float2 qf = ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]; ++#endif ++ sum += k0 * qf.x + k1 * qf.y; ++ } ++ } ++ return sum; ++} ++ ++// Wrapper using __constant__ codebook (for function pointer dispatch) ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2_tcq( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, ++ const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { ++ return vec_dot_fattn_vec_KQ_turbo2_tcq_cb(K_c, Q_v, Q_q8, Q_ds_v, d_turbo2_tcq_codebook_fattn); ++} ++ ++// TCQ 3-bit V dequant: 9-bit state → codebook lookup ++// Core implementation takes explicit codebook pointer for SMEM/constant flexibility ++template ++static __device__ __forceinline__ void dequantize_V_turbo3_tcq_cb( ++ const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, ++ const float * __restrict__ cb) { ++ const block_turbo3_tcq * x = (const block_turbo3_tcq *) vx; ++ const int64_t ib = i0 / QK_TURBO3_TCQ; ++ const int j0 = (int)(i0 % QK_TURBO3_TCQ); ++ const float norm = __half2float(x[ib].norm) * d_tcq_decode_alpha_v_fattn; ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ float vals[ne]; ++#pragma unroll ++ for (int l = 0; l < ne; l++) { ++ const int t = j0 + l; ++ const int bit_pos = t * 3; ++ const uint16_t raw = (uint16_t)x[ib].qs[bit_pos/8] | ((uint16_t)x[ib].qs[bit_pos/8 + 1] << 8); ++ const int state = (raw >> (bit_pos % 8)) & 0x1FF; ++ vals[l] = cb[state] * norm; ++ } ++#ifdef FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ for (int l0 = 0; l0 < ne; l0 += 2) ++ ((half2 *)dst)[l0/2] = make_half2(__float2half(vals[l0]), __float2half(vals[l0+1])); ++ } else ++#endif ++ if constexpr (std::is_same_v) { ++ for (int l = 0; l < ne; ++l) ((float *)dst)[l] = vals[l]; ++ } else { static_assert(std::is_same_v, "bad type"); } ++} ++ ++// Wrapper using __constant__ codebook (for function pointer dispatch via dequantize_V_t) ++template ++static __device__ __forceinline__ void dequantize_V_turbo3_tcq( ++ const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ dequantize_V_turbo3_tcq_cb(vx, dst, i0, d_turbo3_tcq_codebook_fattn); ++} ++ ++// TCQ 2-bit V dequant: 8-bit state → codebook lookup ++template ++static __device__ __forceinline__ void dequantize_V_turbo2_tcq_cb( ++ const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, ++ const float * __restrict__ cb) { ++ const block_turbo2_tcq * x = (const block_turbo2_tcq *) vx; ++ const int64_t ib = i0 / QK_TURBO2_TCQ; ++ const int j0 = (int)(i0 % QK_TURBO2_TCQ); ++ const float norm = __half2float(x[ib].norm) * d_tcq_decode_alpha_v_fattn; ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ float vals[ne]; ++#pragma unroll ++ for (int l = 0; l < ne; l++) { ++ const int t = j0 + l; ++ const int bit_pos = t * 2; ++ const uint16_t raw = (uint16_t)x[ib].qs[bit_pos/8] | ((uint16_t)x[ib].qs[bit_pos/8 + 1] << 8); ++ const int state = (raw >> (bit_pos % 8)) & 0xFF; ++ vals[l] = cb[state] * norm; ++ } ++#ifdef FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ for (int l0 = 0; l0 < ne; l0 += 2) ++ ((half2 *)dst)[l0/2] = make_half2(__float2half(vals[l0]), __float2half(vals[l0+1])); ++ } else ++#endif ++ if constexpr (std::is_same_v) { ++ for (int l = 0; l < ne; ++l) ((float *)dst)[l] = vals[l]; ++ } else { static_assert(std::is_same_v, "bad type"); } ++} ++ ++// Wrapper using __constant__ codebook (for function pointer dispatch via dequantize_V_t) ++template ++static __device__ __forceinline__ void dequantize_V_turbo2_tcq( ++ const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { ++ dequantize_V_turbo2_tcq_cb(vx, dst, i0, d_turbo2_tcq_codebook_fattn); ++} ++ +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +index b72c00bd9..abcce887c 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -79,7 +79,11 @@ static __global__ void flash_attn_ext_vec( + // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Turbo K reads centroid×norm + // against a FLOAT Q (Q_q8_1=false below) and its vec_dot mirrors the f16 path, so it takes the + // f16-style KQ thread split. Turbo V uses the quantized split (nthreads_V_q, V_rows_per_thread=4). +- constexpr bool is_turbo_K = (type_K == GGML_TYPE_TURBO2_0 || type_K == GGML_TYPE_TURBO3_0 || type_K == GGML_TYPE_TURBO4_0); ++ constexpr bool is_turbo_K = (type_K == GGML_TYPE_TURBO2_0 || type_K == GGML_TYPE_TURBO3_0 || type_K == GGML_TYPE_TURBO4_0 ++ // opencoti-hook: TCQ FA-VEC decode (#447/#500). TCQ K is FWHT-rotated like turbo_0 — it must take ++ // the f16-style KQ split AND keep Q FLOAT (Q_q8_1=false below); q8_1-quantizing Q destroys the ++ // WHT-rotation precision the trellis reconstruction needs. V uses our quantized split (ne=4), same as turbo_0. ++ || type_K == GGML_TYPE_TURBO3_TCQ || type_K == GGML_TYPE_TURBO2_TCQ); + constexpr int nthreads = ggml_cuda_fattn_vec_get_nthreads_device(); + constexpr int nthreads_KQ = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16 || is_turbo_K) ? 128 / cpy_nb : nthreads_KQ_q; + constexpr int nthreads_V = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_V_q; +@@ -588,6 +592,7 @@ void ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_ten + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_0); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_1); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_0); \ ++ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q6_0); /* opencoti-hook: q6_0 KV (Anbeeld port) */ \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \ + extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_BF16); \ +@@ -596,6 +601,7 @@ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_1) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0) ++EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q6_0) // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0) + EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_BF16) +@@ -604,6 +610,7 @@ EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_1) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0) ++EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q6_0) // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0) + EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_BF16) +@@ -612,6 +619,7 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_1) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0) ++EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q6_0) // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0) + EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) +@@ -626,11 +634,27 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_0) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_0) + EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO4_0) ++// opencoti-hook: TCQ FA-VEC decode (#447/#500) — trellis-coded turbo K==V vec instances (64/128/256). ++EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_TCQ) ++EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_TCQ) ++// opencoti-hook: asymmetric TCQ (#513) — turbo3_tcq/turbo2_tcq + reverse, head_dim {64,128,256}. ++extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); ++extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); ++extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); + + // opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY — Gemma-4 GLOBAL + // layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256. + extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); + extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); ++// opencoti-hook: TCQ FA-VEC decode (#447/#500) — head_dim 512 trellis VEC for Gemma-4 GLOBAL layers. ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ); ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ); ++// opencoti-hook: asymmetric TCQ (#513) — head_dim 512 (Gemma-4 GLOBAL), strong-K + reverse. ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); + + // opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC — Gemma-4 GLOBAL layer at gqa_ratio<=2 (the E4B + // assistant-MTP drafter), where MMA/TILE abort (DKQ>256 && gqa<=2, bug-567). The VEC kernel tiles +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index c0a964d29..a48785e89 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -347,6 +347,18 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) + #endif // GGML_CUDA_FA_ALL_QUANTS + ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). The Anbeeld instance set (the 5 K/V combos ++ // ik_llama.cpp ships for q6_0 KV), registered OUTSIDE the FA_ALL_QUANTS block so they compile in ++ // the DEFAULT build — the matching instance .cu files are added unconditionally in ++ // collect_gpu_sources (build-functions.sh section 3c), like the turbo K==V instances below. ++ // q6_0-q6_0 is the symmetric (default-reachable) cell; the 4 mixed cells are reachable only under ++ // FA_ALL_QUANTS (the K!=V early-out in ggml_cuda_get_best_fattn_kernel), same as q5_0's mixed pairs. ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q6_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q6_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q5_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_F16) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q6_0) ++ + // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec path + // (always registered, independent of GGML_CUDA_FA_ALL_QUANTS). turbo2/3/4 at head_dim ∈ + // {64,128,256}; turbo8 / head_dim 512 are NOT here (Level-A materialize lift handles them). +@@ -354,11 +366,26 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0) + ++ // opencoti-hook: TCQ FA-VEC decode (#447/#500) — trellis turbo K==V at head_dim {64,128,256}. ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ) ++ // opencoti-hook: asymmetric TCQ (#513) — turbo3_tcq K / turbo2_tcq V (strong-K) + reverse, head_dim {64,128,256}. ++ // K!=V both-TCQ: graph fuses (build_attn_mha) + selector K!=V gate relaxed above. Codebooks baked per-TU ++ // in the instance files (A′ override via tcq{3,2}_set_decode_codebook_{t3t2,t2t3} in set-rows.cu). ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ) ++ + // opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL + // layers). turbo4/turbo8 keep no D=512 instance (Level-A materialize). Matches the fattn.cu + // kernel-selection relax + the L3 graph gate (both turbo2/3 + D==512 only). + FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0) + FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0) ++ // opencoti-hook: TCQ FA-VEC decode (#447/#500) — head_dim 512 trellis for Gemma-4 GLOBAL layers. ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ) ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ) ++ // opencoti-hook: asymmetric TCQ (#513) — head_dim 512 (Gemma-4 GLOBAL), strong-K + reverse. ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ) + + // opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC dispatch — the runtime counterpart to the + // selector's D512/gqa<=2 f16 VEC route (ggml_cuda_get_best_fattn_kernel) + the f16-f16 D512 +@@ -471,8 +498,14 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + } + + #ifndef GGML_CUDA_FA_ALL_QUANTS +- if (K->type != V->type) { +- return BEST_FATTN_KERNEL_NONE; ++ // opencoti-hook: asymmetric TCQ (#513) — both-TCQ K!=V (turbo3_tcq/turbo2_tcq + reverse) has a ++ // dedicated in-register FA-VEC instance, so let it through; every other K!=V still falls back. ++ { ++ const bool both_tcq = (K->type == GGML_TYPE_TURBO3_TCQ || K->type == GGML_TYPE_TURBO2_TCQ) && ++ (V->type == GGML_TYPE_TURBO3_TCQ || V->type == GGML_TYPE_TURBO2_TCQ); ++ if (K->type != V->type && !both_tcq) { ++ return BEST_FATTN_KERNEL_NONE; ++ } + } + #endif // GGML_CUDA_FA_ALL_QUANTS + +@@ -488,11 +521,16 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + #endif // GGML_CUDA_FA_ALL_QUANTS + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q8_0: ++ case GGML_TYPE_Q6_0: // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — the 5 Anbeeld ++ // q6_0 vec instances compile in the DEFAULT build (collect_gpu_sources ++ // section 3c), so q6_0 K is allowed unconditionally like q4_0/q8_0/bf16. + case GGML_TYPE_BF16: + break; + case GGML_TYPE_TURBO2_0: // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392 + case GGML_TYPE_TURBO3_0: + case GGML_TYPE_TURBO4_0: ++ case GGML_TYPE_TURBO3_TCQ: // opencoti-hook: TCQ FA-VEC decode (#447/#500) ++ case GGML_TYPE_TURBO2_TCQ: + break; + default: + return BEST_FATTN_KERNEL_NONE; +@@ -510,13 +548,15 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + // VEC kernel (centroid×norm read in-register); never route them to MMA/WMMA/TILE, which can't + // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when + // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift. +- if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0) { ++ if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0 || ++ K->type == GGML_TYPE_TURBO3_TCQ || K->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ FA-VEC decode (#447/#500) + // opencoti #411 (F5 M6-S2 Level-B): turbo2/3 also ship a D=512 fused VEC instance for Gemma-4 + // GLOBAL layers (key_length=512) so the 256k global cache reads in-register instead of the + // Level-A turbo→f16 materialize. turbo4/turbo8 have no D=512 instance → keep the ≤256 cap + // (Level-A fallback at 512). The L3 graph gate only emits turbo2/3 K/V to FA at 512 to match. + const bool turbo512_vec = Q->ne[0] == 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 && +- (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0); ++ (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || ++ K->type == GGML_TYPE_TURBO3_TCQ || K->type == GGML_TYPE_TURBO2_TCQ); // opencoti-hook: TCQ #447/#500 (D512 trellis) + return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE; + } + +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +index 672bdb5ab..a6ef9eef8 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -5391,8 +5391,12 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || + op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL || ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — KV-write dst (set-rows.cu generic quant kernel) ++ op->type == GGML_TYPE_Q6_0 || + op->type == GGML_TYPE_TURBO3_0 || op->type == GGML_TYPE_TURBO4_0 || +- op->type == GGML_TYPE_TURBO2_0 || op->type == GGML_TYPE_TURBO8_0) && ++ op->type == GGML_TYPE_TURBO2_0 || op->type == GGML_TYPE_TURBO8_0 || ++ // opencoti-hook: TCQ (F5 M6 C5) — turbo3_tcq/turbo2_tcq Viterbi encode (set-rows.cu) ++ op->type == GGML_TYPE_TURBO3_TCQ || op->type == GGML_TYPE_TURBO2_TCQ) && + op->src[0]->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); + } break; +@@ -5445,6 +5449,13 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + if (src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_F32) { + return true; + } ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — f32<->q6_0 cpy (mirrors q5_0). ++ if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q6_0) { ++ return true; ++ } ++ if (src0_type == GGML_TYPE_Q6_0 && src1_type == GGML_TYPE_F32) { ++ return true; ++ } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) { + return true; + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +index 3e4a76076..f0bb7cba2 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +@@ -3,6 +3,9 @@ + #include "turbo-dequant.cuh" // opencoti-hook: turboquant — see docs/features/poly_kv.md (M6-S2 Gate-2 write path) + #include "turbo-innerq.cuh" // opencoti-hook: turboquant InnerQ (M6-S2) — per-channel equalization + #include "turbo-innerq-dev.cuh" // opencoti-hook: turboquant perf-lift (M6-S2 Level A) — cross-TU device-scale accessor ++#include "turbo-tcq-cuda.cuh" // opencoti-hook: TCQ (F5 M6 C5) — Viterbi encode kernels + codebooks + GET_ROWS dequant ++#include "tcq-decode-cb.cuh" // opencoti-hook: TCQ A′ — cross-TU decode-codebook setters (symmetric override) ++#include // strncmp (cache_k_ vs cache_v_ detection for TCQ alpha) + #include + #include + +@@ -598,6 +601,272 @@ static __global__ void k_set_rows_turbo3( + if (elem_in_block == 0) blk->norm = __float2half(corrected_norm); + } + ++// ===================== TCQ (F5 M6 C5): Trellis-Coded Quantization encode dispatch ===================== ++// opencoti-hook: TCQ — drives the buun Viterbi encode kernels (turbo-tcq-cuda.cuh). Ported from buun's ++// ggml_cuda_op_set_rows TURBO*_TCQ branches, adapted to our per-type helper style. The fp32 path is the ++// correctness anchor (byte-faithful to buun); fp64-selectable encode lands as a follow-on. ++ ++// Per-device Viterbi backtrace fallback buffer. Only used when the 8 KB/block shared-memory backtrace is ++// unavailable; on cc>=7.5 (e.g. RTX 3090, 99 KB optin) the shared-bt path is the default, so this rarely allocs. ++static uint8_t * tcq_bt_buf [GGML_CUDA_MAX_DEVICES] = {}; ++static size_t tcq_bt_buf_sz[GGML_CUDA_MAX_DEVICES] = {}; ++static void ensure_tcq_bt_buf(int device, size_t bytes) { ++ if (tcq_bt_buf_sz[device] >= bytes) return; ++ if (tcq_bt_buf[device]) CUDA_CHECK(cudaFree(tcq_bt_buf[device])); ++ CUDA_CHECK(cudaMalloc((void**)&tcq_bt_buf[device], bytes)); ++ tcq_bt_buf_sz[device] = bytes; ++} ++ ++// Context-adaptive norm-scaling override. Sets the turbo-tcq-cuda.cuh __constant__ symbols for THIS TU. ++// Env now (TURBO_TCQ_ALPHA / _ALPHA_V); Stage-4 replaces with the closed-form n_kv schedule. ++static void load_tcq_norm_alpha(int device) { ++ (void) device; ++ const char * sa = getenv("TURBO_TCQ_ALPHA"); ++ const char * sv = getenv("TURBO_TCQ_ALPHA_V"); ++ if (sa) { float a = (float) atof(sa); CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_norm_alpha, &a, sizeof(float))); } ++ if (sv) { float a = (float) atof(sv); CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_norm_alpha_v, &a, sizeof(float))); } ++} ++ ++// opencoti-hook: TCQ fp64-selectable encode — see docs/evaluations/context.md §5a-TCQ. When TURBO_TCQ_FP64=1, ++// the host dispatches the instantiation of the Viterbi encode kernels (acc_t=double), so the ++// path-cost accumulation + norm/whiten reductions run in fp64 — the "true optimal path" oracle for the ++// precision-sensitivity question. Default off ⇒ acc_t=float ⇒ byte-identical to the buun fp32 trellis. ++// Process-global (the env is read once); decode stays fp32 (it reads stored integer states). ++static bool tcq_fp64_enabled() { ++ static int v = -1; ++ if (v < 0) { ++ const char * e = getenv("TURBO_TCQ_FP64"); ++ v = (e && atoi(e) != 0) ? 1 : 0; ++ if (v) fprintf(stderr, "TCQ encode: fp64 Viterbi path ENABLED (TURBO_TCQ_FP64=1)\n"); ++ } ++ return v != 0; ++} ++ ++// opencoti-hook: TCQ activation-dump (F-D, plan lexical-gliding-flurry / #515) — see docs/evaluations/context.md §5a-TCQ. ++// Host side of the dump that k_set_rows_turbo3_tcq's d_tcq_dump_buf write feeds. When TURBO_TCQ_DUMP_ACTS= ++// is set, after each turbo3_tcq encode launch we copy the staged post-FWHT unit-norm 128-vecs D2H and append them ++// as raw little-endian float32 (128 floats/vec) to — caller MUST point this at persistent disk, NEVER /tmp. ++// Capped at TURBO_TCQ_DUMP_MAX vecs (default 200000 ≈ 100 MB). These whitened vecs are codebook-resolution- ++// independent (FWHT precedes the trellis), so this base-tier dump trains the 512-entry turbo3_tcq F-E ++// codebook. Zero overhead when the env is unset: the gate short-circuits and d_tcq_dump_buf stays nullptr (the ++// kernel branch is then a uniform no-op). ++struct tcq_dump_state { ++ bool checked = false; ++ bool active = false; ++ FILE * fp = nullptr; ++ long max_vecs = 200000; ++ long written = 0; ++ float * dev = nullptr; // device staging (grow-only); freed at cap ++ long dev_vecs = 0; // device capacity in vecs ++ float * host = nullptr; // pinned host staging (grow-only); freed at cap ++ long host_vecs = 0; ++}; ++static tcq_dump_state g_tcq_dump; ++ ++// One-time: read env + open the append file. Idempotent (guarded by `checked`). ++static void tcq_dump_init() { ++ if (g_tcq_dump.checked) return; ++ g_tcq_dump.checked = true; ++ const char * path = getenv("TURBO_TCQ_DUMP_ACTS"); ++ if (!path || !path[0]) return; ++ const char * mx = getenv("TURBO_TCQ_DUMP_MAX"); ++ if (mx) { long v = atol(mx); if (v > 0) g_tcq_dump.max_vecs = v; } ++ g_tcq_dump.fp = fopen(path, "ab"); // append-binary ++ if (!g_tcq_dump.fp) { fprintf(stderr, "TCQ activation-dump: FAILED to open %s for append\n", path); return; } ++ g_tcq_dump.active = true; ++ fprintf(stderr, "TCQ activation-dump: ENABLED -> %s (cap %ld vecs, 128 f32 each)\n", path, g_tcq_dump.max_vecs); ++} ++ ++// Per-launch (BEFORE the kernel): grow the device+host staging to hold `groups` vecs and return the device pointer ++// to bind to d_tcq_dump_buf, or nullptr when the dump is inactive / cap reached (caller then leaves the symbol null). ++static float * tcq_dump_pre_launch(long groups) { ++ if (!g_tcq_dump.active || groups <= 0 || g_tcq_dump.written >= g_tcq_dump.max_vecs) return nullptr; ++ if (g_tcq_dump.dev_vecs < groups) { ++ if (g_tcq_dump.dev) CUDA_CHECK(cudaFree(g_tcq_dump.dev)); ++ CUDA_CHECK(cudaMalloc((void**)&g_tcq_dump.dev, (size_t)groups * 128 * sizeof(float))); ++ g_tcq_dump.dev_vecs = groups; ++ } ++ if (g_tcq_dump.host_vecs < groups) { ++ if (g_tcq_dump.host) CUDA_CHECK(cudaFreeHost(g_tcq_dump.host)); ++ CUDA_CHECK(cudaMallocHost((void**)&g_tcq_dump.host, (size_t)groups * 128 * sizeof(float))); ++ g_tcq_dump.host_vecs = groups; ++ } ++ return g_tcq_dump.dev; ++} ++ ++// Per-launch (AFTER the kernel): copy the staged vecs D2H on `stream`, append (capped) to the file, advance the ++// counter. At the cap, null the symbol + free staging so further launches are zero-overhead no-ops. ++static void tcq_dump_post_launch(long groups, cudaStream_t stream) { ++ if (!g_tcq_dump.active || g_tcq_dump.dev == nullptr || groups <= 0 || g_tcq_dump.written >= g_tcq_dump.max_vecs) return; ++ long take = groups; ++ if (g_tcq_dump.written + take > g_tcq_dump.max_vecs) take = g_tcq_dump.max_vecs - g_tcq_dump.written; ++ CUDA_CHECK(cudaMemcpyAsync(g_tcq_dump.host, g_tcq_dump.dev, (size_t)take * 128 * sizeof(float), ++ cudaMemcpyDeviceToHost, stream)); ++ CUDA_CHECK(cudaStreamSynchronize(stream)); ++ fwrite(g_tcq_dump.host, sizeof(float), (size_t)take * 128, g_tcq_dump.fp); ++ g_tcq_dump.written += take; ++ if (g_tcq_dump.written >= g_tcq_dump.max_vecs) { ++ fflush(g_tcq_dump.fp); ++ fprintf(stderr, "TCQ activation-dump: cap reached (%ld vecs) -> closing %s\n", g_tcq_dump.written, "dump file"); ++ float * nullp = nullptr; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_dump_buf, &nullp, sizeof(float*))); ++ if (g_tcq_dump.dev) { CUDA_CHECK(cudaFree(g_tcq_dump.dev)); g_tcq_dump.dev = nullptr; g_tcq_dump.dev_vecs = 0; } ++ if (g_tcq_dump.host) { CUDA_CHECK(cudaFreeHost(g_tcq_dump.host)); g_tcq_dump.host = nullptr; g_tcq_dump.host_vecs = 0; } ++ g_tcq_dump.active = false; ++ } ++} ++ ++template ++static void set_rows_cuda_turbo3_tcq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO3_TCQ == 0); ++ cudaStream_t stream = ctx.stream(); ++ const int64_t ne_total_groups = (ne00 * ne01 * ne02 * ne03) / QK_TURBO3_TCQ; ++ if (ne_total_groups <= 0 || !(ne00 > 0 && ne01 > 0 && ne02 > 0 && ne11 > 0 && ne12 > 0)) return; ++ ++ // one-time per-device: optional codebook override (TURBO_TCQ_CB) + adaptive-alpha env ++ static bool tcq3_cb_loaded[GGML_CUDA_MAX_DEVICES] = {}; ++ if (!tcq3_cb_loaded[ctx.device]) { ++ tcq3_cb_loaded[ctx.device] = true; ++ const char *cb_path = getenv("TURBO_TCQ_CB"); ++ if (cb_path) { ++ float cb[512]; FILE *f = fopen(cb_path, "rb"); ++ if (f && fread(cb, sizeof(float), 512, f) == 512) { fclose(f); CUDA_CHECK(cudaMemcpyToSymbol(d_turbo3_tcq_codebook, cb, 512*sizeof(float))); ++ tcq3_set_decode_codebook(cb); // opencoti-hook: TCQ A′ — symmetric (encode+decode) override ++ // opencoti-hook: asymmetric TCQ (#513) — also override the per-TU d_turbo3 copies in the ++ // two asymmetric instance TUs so the runtime codebook reaches asym decode (not just symmetric). ++ tcq3_set_decode_codebook_t3t2(cb); tcq3_set_decode_codebook_t2t3(cb); ++ fprintf(stderr, "TCQ encode+decode: loaded codebook from %s (device %d)\n", cb_path, ctx.device); } ++ else { if (f) fclose(f); fprintf(stderr, "TCQ encode: FAILED to load codebook from %s\n", cb_path); } ++ } ++ load_tcq_norm_alpha(ctx.device); ++ } ++ ++ const int64_t s01_f = nb01/sizeof(float), s02_f = nb02/sizeof(float), s03_f = nb03/sizeof(float); ++ const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t); ++ const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0; ++ ++ // one-time per-device: prefer the 8 KB/block shared-memory backtrace (opt-in); else global fallback buffer ++ static int tcq3_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {}; ++ static bool tcq3_bt_checked [GGML_CUDA_MAX_DEVICES] = {}; ++ constexpr int tcq3_bt_shared_bytes = 128 * 64; ++ if (!tcq3_bt_checked[ctx.device]) { ++ tcq3_bt_checked[ctx.device] = true; ++ const char * env = getenv("TURBO_TCQ_SHARED_BT"); ++ if (!env || atoi(env) != 0) { ++ int max_shared_optin = 0; ++ CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device)); ++ if (max_shared_optin >= tcq3_bt_shared_bytes) { ++ if (tcq_fp64_enabled()) ++ CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo3_tcq), tcq3_bt_shared_bytes); ++ else ++ CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo3_tcq, tcq3_bt_shared_bytes); ++ tcq3_use_shared_bt[ctx.device] = 1; ++ } ++ } ++ } ++ if (!tcq3_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); ++ ++ const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00); ++ const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01); ++ const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02); ++ const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); ++ const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); ++ const int shared_bytes = tcq3_use_shared_bt[ctx.device] ? tcq3_bt_shared_bytes : 0; ++ // opencoti-hook: TCQ activation-dump (F-D, #515) — bind the staging buffer for this launch (no-op unless ++ // TURBO_TCQ_DUMP_ACTS is set). tcq_dump_init() is idempotent; dump_dev is nullptr when inactive/cap-reached, ++ // in which case d_tcq_dump_buf is left at its nullptr default and the kernel's dump branch is a uniform no-op. ++ tcq_dump_init(); ++ float * dump_dev = tcq_dump_pre_launch((long) ne_total_groups); ++ if (dump_dev) CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_dump_buf, &dump_dev, sizeof(float*))); ++ if (tcq_fp64_enabled()) ++ k_set_rows_turbo3_tcq<<<(int)ne_total_groups, 512, shared_bytes, stream>>>( ++ src0_d, src1_d, (block_turbo3_tcq *)dst->data, ++ ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, ++ ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); ++ else ++ k_set_rows_turbo3_tcq<<<(int)ne_total_groups, 512, shared_bytes, stream>>>( ++ src0_d, src1_d, (block_turbo3_tcq *)dst->data, ++ ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, ++ ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); ++ if (dump_dev) tcq_dump_post_launch((long) ne_total_groups, stream); ++} ++ ++template ++static void set_rows_cuda_turbo2_tcq(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ++ const float * src0_d = (const float *)src0->data; ++ const idx_t * src1_d = (const idx_t *)src1->data; ++ GGML_TENSOR_BINARY_OP_LOCALS ++ GGML_ASSERT(ne00 % QK_TURBO2_TCQ == 0); ++ cudaStream_t stream = ctx.stream(); ++ const int64_t ne_total_groups = (ne00 * ne01 * ne02 * ne03) / QK_TURBO2_TCQ; ++ if (ne_total_groups <= 0 || !(ne00 > 0 && ne01 > 0 && ne02 > 0 && ne11 > 0 && ne12 > 0)) return; ++ ++ static bool tcq2_cb_loaded[GGML_CUDA_MAX_DEVICES] = {}; ++ if (!tcq2_cb_loaded[ctx.device]) { ++ tcq2_cb_loaded[ctx.device] = true; ++ const char *cb_path = getenv("TURBO_TCQ_CB2"); ++ if (cb_path) { ++ float cb[256]; FILE *f = fopen(cb_path, "rb"); ++ if (f && fread(cb, sizeof(float), 256, f) == 256) { fclose(f); CUDA_CHECK(cudaMemcpyToSymbol(d_turbo2_tcq_codebook, cb, 256*sizeof(float))); ++ tcq2_set_decode_codebook(cb); // opencoti-hook: TCQ A′ — symmetric (encode+decode) override ++ // opencoti-hook: asymmetric TCQ (#513) — also override the per-TU d_turbo2 copies in the ++ // two asymmetric instance TUs so the runtime 2-bit codebook reaches asym decode too. ++ tcq2_set_decode_codebook_t3t2(cb); tcq2_set_decode_codebook_t2t3(cb); ++ fprintf(stderr, "TCQ2 encode+decode: loaded 2-bit codebook from %s (device %d)\n", cb_path, ctx.device); } ++ else { if (f) fclose(f); fprintf(stderr, "TCQ2 encode: FAILED to load codebook from %s\n", cb_path); } ++ } ++ load_tcq_norm_alpha(ctx.device); ++ } ++ ++ const int64_t s01_f = nb01/sizeof(float), s02_f = nb02/sizeof(float), s03_f = nb03/sizeof(float); ++ const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t); ++ const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0; ++ ++ static int tcq2_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {}; ++ static bool tcq2_bt_checked [GGML_CUDA_MAX_DEVICES] = {}; ++ constexpr int tcq2_bt_shared_bytes = 128 * 64; ++ if (!tcq2_bt_checked[ctx.device]) { ++ tcq2_bt_checked[ctx.device] = true; ++ const char * env = getenv("TURBO_TCQ_SHARED_BT"); ++ if (!env || atoi(env) != 0) { ++ int max_shared_optin = 0; ++ CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device)); ++ if (max_shared_optin >= tcq2_bt_shared_bytes) { ++ if (tcq_fp64_enabled()) ++ CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo2_tcq), tcq2_bt_shared_bytes); ++ else ++ CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo2_tcq, tcq2_bt_shared_bytes); ++ tcq2_use_shared_bt[ctx.device] = 1; ++ } ++ } ++ } ++ if (!tcq2_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); ++ ++ const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00); ++ const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01); ++ const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02); ++ const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); ++ const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); ++ const int shared_bytes = tcq2_use_shared_bt[ctx.device] ? tcq2_bt_shared_bytes : 0; ++ if (tcq_fp64_enabled()) ++ k_set_rows_turbo2_tcq<<<(int)ne_total_groups, 256, shared_bytes, stream>>>( ++ src0_d, src1_d, (block_turbo2_tcq *)dst->data, ++ ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, ++ ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); ++ else ++ k_set_rows_turbo2_tcq<<<(int)ne_total_groups, 256, shared_bytes, stream>>>( ++ src0_d, src1_d, (block_turbo2_tcq *)dst->data, ++ ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, ++ ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); ++} ++ + template + static void set_rows_cuda_turbo3(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const float * src0_d = (const float *)src0->data; +@@ -1094,6 +1363,18 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s + nb1, nb2, nb3, + stream + ); ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — KV-write via the generic quantized set_rows ++ // template (mirrors q5_0); quantize_f32_q6_0_block is in cpy-utils.cuh. See docs/features/poly_kv.md. ++ } else if (dst->type == GGML_TYPE_Q6_0) { ++ set_rows_cuda_quant( ++ src0_d, src1_d, (block_q6_0*)dst->data, ++ ne00, ne01, ne02, ne03, ++ ne10, ne11, ne12, ne13, ++ nb01, nb02, nb03, ++ nb10, nb11, nb12, ++ nb1, nb2, nb3, ++ stream ++ ); + } else if (dst->type == GGML_TYPE_Q8_0) { + set_rows_cuda_quant( + src0_d, src1_d, (block_q8_0*)dst->data, +@@ -1126,6 +1407,12 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s + } else if (dst->type == GGML_TYPE_TURBO8_0) { + // opencoti-hook: turboquant tier family — see docs/features/poly_kv.md (M6-S2) + set_rows_cuda_turbo8(ctx, src0, src1, dst); ++ } else if (dst->type == GGML_TYPE_TURBO3_TCQ) { ++ // opencoti-hook: TCQ (F5 M6 C5) — Viterbi 3-bit trellis encode (turbo-tcq-cuda.cuh) ++ set_rows_cuda_turbo3_tcq(ctx, src0, src1, dst); ++ } else if (dst->type == GGML_TYPE_TURBO2_TCQ) { ++ // opencoti-hook: TCQ (F5 M6 C5) — Viterbi 2-bit trellis encode (turbo-tcq-cuda.cuh) ++ set_rows_cuda_turbo2_tcq(ctx, src0, src1, dst); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh b/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh +new file mode 100644 +index 000000000..5819745b9 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh +@@ -0,0 +1,24 @@ ++#pragma once ++// opencoti-hook: TCQ A′ symmetric codebook override — see docs/evaluations/context.md §5a-TCQ. ++// ++// The decode-side TCQ codebooks (d_turbo{3,2}_tcq_codebook_fattn) live as `static __constant__` in ++// fattn-tcq.cuh and are replicated PER-TU (the CUDA DSO compiles each .cu separately, no -rdc). The ++// kernel that READS each copy is the FA-VEC decode instantiated in ++// template-instances/fattn-vec-instance-turbo{3,2}_tcq-*.cu, so the only copy that matters is the one ++// in THAT translation unit. These setters are therefore DEFINED in those instance TUs (where the live ++// copy + cudaMemcpyToSymbol can reach it) and DECLARED here for set-rows.cu's TURBO_TCQ_CB / CB2 loader ++// to call. Without this, TURBO_TCQ_CB overrides only the ENCODE symbol and FA decode keeps the baked ++// codebook → mismatch corruption (footgun A′). With it, a runtime codebook override is symmetric. ++void tcq3_set_decode_codebook(const float * cb); // 512 floats (turbo3_tcq) ++void tcq2_set_decode_codebook(const float * cb); // 256 floats (turbo2_tcq) ++ ++// opencoti-hook: asymmetric TCQ (#513). The asymmetric instance TUs (turbo3_tcq/turbo2_tcq + reverse) ++// each own their OWN per-TU copies of BOTH d_turbo3 + d_turbo2 codebooks (the asym decode reads both: K ++// from one tier, V from the other), so each TU needs its own setter pair — uniquely named to avoid ++// duplicate-symbol link errors with the symmetric setters above. set-rows.cu calls all of these in the ++// TURBO_TCQ_CB / CB2 apply so a runtime override reaches the asymmetric decode TUs too (else they keep ++// the baked codebook while the symmetric path uses the override → inconsistent KLD). ++void tcq3_set_decode_codebook_t3t2(const float * cb); // turbo3_tcq-K / turbo2_tcq-V TU: its d_turbo3 (512) ++void tcq2_set_decode_codebook_t3t2(const float * cb); // turbo3_tcq-K / turbo2_tcq-V TU: its d_turbo2 (256) ++void tcq3_set_decode_codebook_t2t3(const float * cb); // turbo2_tcq-K / turbo3_tcq-V TU: its d_turbo3 (512) ++void tcq2_set_decode_codebook_t2t3(const float * cb); // turbo2_tcq-K / turbo3_tcq-V TU: its d_turbo2 (256) +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q6_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q6_0.cu +new file mode 100644 +index 000000000..1e81a509a +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q6_0.cu +@@ -0,0 +1,7 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Mirrors fattn-vec-instance-f16-q5_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-f16.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-f16.cu +new file mode 100644 +index 000000000..0f2c5b89c +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-f16.cu +@@ -0,0 +1,7 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Mirrors fattn-vec-instance-q5_0-f16.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_F16); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_F16); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_F16); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q5_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q5_0.cu +new file mode 100644 +index 000000000..93a414520 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q5_0.cu +@@ -0,0 +1,7 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Mirrors fattn-vec-instance-q5_0-q4_0.cu form. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_Q5_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_Q5_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q6_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q6_0.cu +new file mode 100644 +index 000000000..04ac8a57d +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q6_0.cu +@@ -0,0 +1,7 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Mirrors fattn-vec-instance-q5_0-q5_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q6_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q6_0.cu +new file mode 100644 +index 000000000..563f54cc4 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q6_0.cu +@@ -0,0 +1,7 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). Mirrors fattn-vec-instance-q8_0-q5_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo2_tcq.cu +new file mode 100644 +index 000000000..9f4e7fbc1 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo2_tcq.cu +@@ -0,0 +1,17 @@ ++// This file mirrors the autogenerated turbo instance files; do not edit manually. ++// opencoti-hook: TCQ (trellis-coded quant) FA-VEC decode — F5 M6 C5 (#447/#500). See UPSTREAM_SYNC.md. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO2_TCQ); // opencoti TCQ: Gemma-4 GLOBAL head_dim 512 ++ ++// opencoti-hook: TCQ A′ symmetric codebook override (decode side) — see tcq-decode-cb.cuh. ++// Defined HERE because d_turbo2_tcq_codebook_fattn is static __constant__ per-TU (this TU owns the ++// live copy the FA-VEC decode above reads). ++#include "../tcq-decode-cb.cuh" ++void tcq2_set_decode_codebook(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo2_tcq_codebook_fattn, cb, 256 * sizeof(float))); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo3_tcq.cu +new file mode 100644 +index 000000000..b51a16283 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_tcq-turbo3_tcq.cu +@@ -0,0 +1,22 @@ ++// This file mirrors the autogenerated turbo instance files; do not edit manually. ++// opencoti-hook: asymmetric TCQ (#513) FA-VEC decode — turbo2_tcq K / turbo3_tcq V (strong-V). ++// See UPSTREAM_SYNC.md and docs/evaluations/context.md §5a-TCQ. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ); // opencoti TCQ: Gemma-4 GLOBAL head_dim 512 ++ ++// opencoti-hook: asymmetric TCQ A′ codebook override (decode side) — this TU owns its OWN per-TU static ++// __constant__ d_turbo3_tcq_codebook_fattn (V decode) AND d_turbo2_tcq_codebook_fattn (K decode), so BOTH ++// need their own setter here (uniquely named vs the symmetric TUs to avoid duplicate symbols). set-rows.cu ++// calls these from the TURBO_TCQ_CB / CB2 loader so a runtime override reaches this TU's decode too. ++#include "../tcq-decode-cb.cuh" ++void tcq3_set_decode_codebook_t2t3(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo3_tcq_codebook_fattn, cb, 512 * sizeof(float))); ++} ++void tcq2_set_decode_codebook_t2t3(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo2_tcq_codebook_fattn, cb, 256 * sizeof(float))); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo2_tcq.cu +new file mode 100644 +index 000000000..da175f9c1 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo2_tcq.cu +@@ -0,0 +1,22 @@ ++// This file mirrors the autogenerated turbo instance files; do not edit manually. ++// opencoti-hook: asymmetric TCQ (#513) FA-VEC decode — turbo3_tcq K / turbo2_tcq V (strong-K). ++// See UPSTREAM_SYNC.md and docs/evaluations/context.md §5a-TCQ. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ); // opencoti TCQ: Gemma-4 GLOBAL head_dim 512 ++ ++// opencoti-hook: asymmetric TCQ A′ codebook override (decode side) — this TU owns its OWN per-TU static ++// __constant__ d_turbo3_tcq_codebook_fattn (K decode) AND d_turbo2_tcq_codebook_fattn (V decode), so BOTH ++// need their own setter here (uniquely named vs the symmetric TUs to avoid duplicate symbols). set-rows.cu ++// calls these from the TURBO_TCQ_CB / CB2 loader so a runtime override reaches this TU's decode too. ++#include "../tcq-decode-cb.cuh" ++void tcq3_set_decode_codebook_t3t2(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo3_tcq_codebook_fattn, cb, 512 * sizeof(float))); ++} ++void tcq2_set_decode_codebook_t3t2(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo2_tcq_codebook_fattn, cb, 256 * sizeof(float))); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo3_tcq.cu +new file mode 100644 +index 000000000..906910dde +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_tcq-turbo3_tcq.cu +@@ -0,0 +1,17 @@ ++// This file mirrors the autogenerated turbo instance files; do not edit manually. ++// opencoti-hook: TCQ (trellis-coded quant) FA-VEC decode — F5 M6 C5 (#447/#500). See UPSTREAM_SYNC.md. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO3_TCQ); // opencoti TCQ: Gemma-4 GLOBAL head_dim 512 ++ ++// opencoti-hook: TCQ A′ symmetric codebook override (decode side) — see tcq-decode-cb.cuh. ++// Defined HERE because d_turbo3_tcq_codebook_fattn is static __constant__ per-TU and the live copy the ++// FA-VEC decode above reads is this TU's. cudaMemcpyToSymbol on a static symbol only works in-TU. ++#include "../tcq-decode-cb.cuh" ++void tcq3_set_decode_codebook(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo3_tcq_codebook_fattn, cb, 512 * sizeof(float))); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +new file mode 100644 +index 000000000..f5b3ebba7 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +@@ -0,0 +1,747 @@ ++#pragma once ++// opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV ENCODE (CUDA). ++// Ported from buun (github.com/spiritbuun/buun-llama-cpp, ggml-cuda/turbo-quant-cuda.cuh): ++// the 512/256-state right-shift Viterbi encode kernels, the GET_ROWS dequant helpers, and the ++// trained compiled-in codebooks (credit spiritbuun). Adaptations vs buun (all additive/syncable): ++// * reuses our TURBO3_WHT_SIGNS1/2 (turbo-dequant.cuh) — verified byte-identical to buun's ++// d_turbo_wht_signs1/2, so the trained codebook stays valid; ++// * InnerQ calibrate + per-channel scale blocks STRIPPED (identity) — matches our turbo3 encode ++// ("InnerQ stub-disabled"); avoids colliding with our head_dim-aware d_innerq_* in set-rows.cu. ++// TCQ x InnerQ composition is a deliberate later A/B (our InnerQ is head_dim-aware, buun's 128-only); ++// * diagnostic dump blocks (d_tcq_dump_*) and turbo_extract_append removed; ++// * fp32 baseline (byte-faithful to buun). fp64-selectable encode (env TURBO_TCQ_FP64) lands as a ++// follow-on once the fp32 trellis is correctness-gated (logit-equiv + KLD vs buun). ++// Provenance + design: docs/evaluations/context.md §5. ++#include "common.cuh" ++#include "turbo-dequant.cuh" // TURBO3_WHT_SIGNS1/2 (reused; byte-identical to buun) ++ ++// NOTE: the context-adaptive norm-scaling symbols d_tcq_norm_alpha (K) / d_tcq_norm_alpha_v (V) ++// are declared in the extracted buun section below (defaults 1.0 / 1.04). Stage-4 tunes them via ++// cudaMemcpyToSymbol from set-rows.cu (TURBO_TCQ_ALPHA / _ALPHA_V env). ++ ++static __constant__ float d_turbo3_tcq_codebook[512] = { ++ -0.14559399f, -0.09062801f, -0.054925077f, -0.03699251f, -0.006363985f, +0.026264573f, +0.067378916f, +0.121981815f, ++ -0.18648055f, -0.106522456f, -0.052047577f, -0.011695214f, +0.021953275f, +0.059698727f, +0.09831437f, +0.16083933f, ++ -0.16390342f, -0.12639847f, -0.09513180f, -0.05938352f, -0.028396897f, +0.005973862f, +0.049104784f, +0.11334257f, ++ -0.25952467f, -0.079778515f, -0.036024813f, +0.0003641268f, +0.031858794f, +0.073280424f, +0.11835553f, +0.19738495f, ++ -0.14218009f, -0.10224814f, -0.062498566f, -0.027066832f, +0.00393002f, +0.04069300f, +0.08257346f, +0.14548601f, ++ -0.18673635f, -0.13438253f, -0.088401966f, -0.05205436f, -0.02032501f, +0.012399545f, +0.05127183f, +0.10316186f, ++ -0.10807011f, -0.065903045f, -0.032206114f, -0.0062006037f, +0.020679146f, +0.04422085f, +0.08313074f, +0.16821936f, ++ -0.22979105f, -0.14431947f, -0.07689272f, -0.02755307f, +0.009225173f, +0.046684854f, +0.08834142f, +0.13766693f, ++ -0.22114082f, -0.12612148f, -0.06890522f, -0.016128855f, +0.03691900f, +0.08474852f, +0.14940020f, +0.23229980f, ++ -0.14933491f, -0.099693604f, -0.06738499f, -0.037100967f, -0.009332986f, +0.023535024f, +0.060272533f, +0.109464675f, ++ -0.20200425f, -0.07398328f, -0.038700905f, -0.01714807f, +0.011161969f, +0.04528101f, +0.08902637f, +0.19573534f, ++ -0.16645233f, -0.124482535f, -0.089342155f, -0.04427387f, -0.007353691f, +0.028033108f, +0.066108435f, +0.15552913f, ++ -0.22295763f, -0.059887577f, -0.018804537f, +0.020141022f, +0.059682943f, +0.097920544f, +0.14080113f, +0.25698325f, ++ -0.14248224f, -0.089685425f, -0.050101686f, -0.017257255f, +0.011412255f, +0.040830314f, +0.07400172f, +0.11997315f, ++ -0.18649384f, -0.113997504f, -0.067775466f, -0.033394672f, +0.006586988f, +0.05312057f, +0.10433043f, +0.22344802f, ++ -0.16138338f, -0.108194515f, -0.07600300f, -0.05135381f, -0.023365447f, +0.0087320795f, +0.045431953f, +0.09113002f, ++ -0.12630440f, -0.07225349f, -0.032280035f, +0.0029231994f, +0.019239848f, +0.05081419f, +0.077840395f, +0.121695265f, ++ -0.08928155f, -0.044983763f, -0.009889568f, +0.020831043f, +0.05684458f, +0.09409702f, +0.13867535f, +0.19084482f, ++ -0.14182915f, -0.11380146f, -0.06904074f, -0.002002765f, +0.034864165f, +0.070399575f, +0.11403063f, +0.15394832f, ++ -0.10876417f, -0.056122433f, -0.02267638f, +0.011113975f, +0.039639056f, +0.074084364f, +0.10155376f, +0.12540291f, ++ -0.17693359f, -0.13940524f, -0.10049578f, -0.06796275f, -0.036915872f, +0.00062823476f, +0.042142134f, +0.17906062f, ++ -0.09253492f, -0.04290128f, -0.006311852f, +0.023908244f, +0.049849935f, +0.078770354f, +0.10818172f, +0.15166481f, ++ -0.12429565f, -0.07392063f, -0.029114135f, +0.0059440783f, +0.042675965f, +0.08425635f, +0.13836108f, +0.18634140f, ++ -0.11795639f, -0.07033707f, -0.034163877f, -0.0008773357f, +0.03334606f, +0.07188203f, +0.12216825f, +0.17097956f, ++ -0.18718453f, -0.14090346f, -0.097799584f, -0.059522875f, -0.019208657f, +0.03079176f, +0.09334672f, +0.15811224f, ++ -0.27198875f, -0.16546582f, -0.11433405f, -0.06933013f, -0.04026183f, -0.0061146915f, +0.029263576f, +0.07322499f, ++ -0.18471734f, -0.102074504f, -0.06492570f, -0.034418534f, -0.009636157f, +0.023043344f, +0.05751496f, +0.09905984f, ++ -0.22826399f, -0.15946552f, -0.09913176f, -0.06585259f, -0.03252090f, +0.001313243f, +0.03556729f, +0.21612854f, ++ -0.13243781f, -0.087299444f, -0.049820945f, -0.016216082f, +0.01799807f, +0.057916876f, +0.09001349f, +0.13221787f, ++ -0.19516511f, -0.120894566f, -0.076130204f, -0.051442243f, -0.029535033f, -0.0020043184f, +0.029452588f, +0.075566076f, ++ -0.27272871f, -0.15841717f, -0.105432935f, -0.06792948f, -0.024532158f, +0.014960791f, +0.054415092f, +0.101517834f, ++ -0.21153601f, -0.15015371f, -0.08676790f, -0.04414934f, -0.0042129597f, +0.033762872f, +0.07589151f, +0.12768789f, ++ -0.090428725f, -0.037582967f, +0.0013173596f, +0.03900247f, +0.06840049f, +0.116906695f, +0.16584939f, +0.25382105f, ++ -0.13446195f, -0.07865091f, -0.039625354f, -0.0028398742f, +0.03019514f, +0.06799379f, +0.11850997f, +0.17521496f, ++ -0.11350345f, -0.058599845f, -0.017512511f, +0.019431496f, +0.055897832f, +0.093173414f, +0.14820710f, +0.22092152f, ++ -0.15165758f, -0.08869354f, -0.04974287f, -0.01705474f, +0.013134752f, +0.04367713f, +0.07733791f, +0.12430801f, ++ -0.09329869f, -0.04673005f, -0.00045857552f, +0.042781368f, +0.07802363f, +0.11887439f, +0.16250038f, +0.28612965f, ++ -0.12571070f, -0.07786012f, -0.03843933f, -0.0075433915f, +0.025822964f, +0.066053316f, +0.12021536f, +0.18341768f, ++ -0.16079275f, -0.04921760f, -0.006114644f, +0.026215268f, +0.05699377f, +0.09813471f, +0.16080129f, +0.23786584f, ++ -0.09980837f, -0.048535258f, -0.0096120685f, +0.025387142f, +0.05979822f, +0.09875251f, +0.14474337f, +0.20324114f, ++ -0.15846540f, -0.09938028f, -0.061492465f, -0.03523542f, -0.0061364113f, +0.024916094f, +0.06037314f, +0.106796466f, ++ -0.20557843f, -0.123237535f, -0.07734871f, -0.044549115f, -0.017114898f, +0.01616654f, +0.049574375f, +0.092319444f, ++ -0.19221115f, -0.14642999f, -0.091701314f, -0.055265956f, -0.021026207f, +0.017720066f, +0.05786183f, +0.110154524f, ++ -0.09956386f, -0.03870283f, +0.003052007f, +0.034851722f, +0.06256365f, +0.09628840f, +0.13979156f, +0.16582295f, ++ -0.18026546f, -0.12448310f, -0.07424377f, -0.03954519f, -0.01221123f, +0.028641058f, +0.100819774f, +0.18240699f, ++ -0.21520759f, -0.15573645f, -0.09820838f, -0.051450998f, -0.012993679f, +0.021135861f, +0.058727216f, +0.105848536f, ++ -0.11207385f, -0.08335689f, -0.048542723f, -0.023198519f, +0.0039304253f, +0.037778318f, +0.07813917f, +0.13106476f, ++ -0.17849164f, -0.120988995f, -0.078016765f, -0.043093704f, -0.016565649f, +0.015182641f, +0.050754096f, +0.09595712f, ++ -0.22132620f, -0.13407415f, -0.065785654f, -0.013291034f, +0.032098345f, +0.07478225f, +0.12431934f, +0.19174045f, ++ -0.095454164f, -0.051898945f, -0.015116375f, -0.012596778f, +0.018636847f, +0.05006925f, +0.087654814f, +0.13754296f, ++ -0.15254061f, -0.09576059f, -0.052086458f, -0.01596074f, +0.017607626f, +0.04778498f, +0.08950204f, +0.14901252f, ++ -0.26057002f, -0.12472382f, -0.074396215f, -0.03764066f, +0.0011168446f, +0.061569117f, +0.10793752f, +0.19771695f, ++ -0.08661132f, -0.045195263f, -0.016098704f, +0.012780116f, +0.040476497f, +0.074102715f, +0.074102715f, +0.12635531f, ++ -0.14047913f, -0.059587404f, -0.016261123f, +0.019801628f, +0.053541403f, +0.096650146f, +0.15005490f, +0.21051759f, ++ -0.22986396f, -0.11964334f, -0.07266585f, -0.026522418f, +0.018169926f, +0.058630653f, +0.100647695f, +0.15919648f, ++ -0.13251697f, -0.077567816f, -0.042766172f, -0.011389967f, +0.01831755f, +0.05304656f, +0.09620367f, +0.15567583f, ++ -0.119819686f, -0.06772876f, -0.028123451f, +0.00876240f, +0.014405836f, +0.048829112f, +0.08422175f, +0.13823749f, ++ -0.16379014f, -0.08956941f, -0.041652776f, +0.008921398f, +0.05473602f, +0.10037984f, +0.16022855f, +0.23457925f, ++ -0.115844205f, -0.05939626f, -0.020390417f, +0.01374377f, +0.044976473f, +0.07873563f, +0.12207942f, +0.18412720f, ++ -0.19048831f, -0.07587487f, -0.03220580f, -0.00011795067f, +0.02721784f, +0.04380719f, +0.07886723f, +0.13193911f, ++ -0.13935551f, -0.092902906f, -0.052706074f, -0.017797327f, +0.015312965f, +0.056098964f, +0.11203423f, +0.24448302f, ++ -0.17986591f, -0.10738580f, -0.06376371f, -0.026595421f, +0.00842492f, +0.04272362f, +0.08608052f, +0.15240218f, ++ -0.10953678f, -0.057022586f, -0.012483291f, +0.024463262f, +0.06076792f, +0.09776234f, +0.12983681f, +0.18648379f, ++ -0.16471463f, -0.089491285f, -0.037574016f, +0.004444791f, +0.039293647f, +0.07845859f, +0.12893885f, +0.23508036f ++}; ++ ++// TCQ norm alpha: K alpha = 1.0 (no scaling), V alpha = 1.04 (optimal at 2K context). ++// Override via TURBO_TCQ_ALPHA (K) and TURBO_TCQ_ALPHA_V (V) env vars. ++// Alpha is applied at encode time (baked into fp16 norm) — this outperforms decode-time application. ++static __constant__ float d_tcq_norm_alpha = 1.0f; ++static __constant__ float d_tcq_norm_alpha_v = 1.04f; ++ ++// opencoti-hook: TCQ activation-dump (F-D, plan lexical-gliding-flurry / #515) — see docs/evaluations/context.md §5a-TCQ. ++// When non-null, k_set_rows_turbo3_tcq stages each group's post-FWHT unit-norm 128-vec (the exact codebook-domain ++// signal the trellis quantizes against d_turbo3_tcq_codebook) into this device buffer at [group*128 + i]. Set ++// per-launch from set-rows.cu ONLY when TURBO_TCQ_DUMP_ACTS is set; stays nullptr otherwise (uniform branch, zero ++// overhead). Feeds F-E's real-27B codebook retrain (the 512-entry turbo3_tcq codebook); the whitened unit-norm ++// vec is codebook-resolution-independent (FWHT happens before the trellis). ++static __constant__ float * d_tcq_dump_buf = nullptr; ++ ++// fp64-selectable encode (env TURBO_TCQ_FP64): the Viterbi forward pass and the ++// norm/whiten reductions run in acc_t. acc_t defaults to float — every existing ++// launch resolves to the float instantiation, byte-identical to buun. ++// The host dispatches the instantiation when TURBO_TCQ_FP64 is set. ++// Overloaded (not specialized) so the float overload lowers to the exact same ++// sqrtf SASS as the original code → fp32 byte-identity is guaranteed by construction. ++static __device__ __forceinline__ float tcq_acc_sqrt(float v) { return sqrtf(v); } ++static __device__ __forceinline__ double tcq_acc_sqrt(double v) { return sqrt(v); } ++ ++// TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis ++// 512 threads per block (one per trellis state), one block per 128-element group ++// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384) ++template ++static __global__ void __launch_bounds__(512, 1) k_set_rows_turbo3_tcq( ++ const float * __restrict__ src0, const idx_t * __restrict__ src1, ++ block_turbo3_tcq * __restrict__ dst, const int64_t ne_total_groups, ++ uint8_t * __restrict__ bt_buf, ++ const int use_shared_bt, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int innerq_is_k, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd, ++ const uint3 ne11_fd, const uint3 ne12_fd) { ++ ++ const int64_t group = blockIdx.x; ++ if (group >= ne_total_groups) return; ++ ++ const int sid = threadIdx.x; // state index 0..511 ++ ++ // Compute source and destination pointers (same index math as turbo3) ++ const int64_t i_base = group * QK_TURBO3_TCQ; ++ uint32_t tmp = (uint32_t)i_base; uint2 div_mod; ++ div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x; ++ div_mod = fast_div_modulo(tmp, ne01_fd); const int64_t i01 = div_mod.y; tmp = div_mod.x; ++ div_mod = fast_div_modulo(tmp, ne02_fd); const int64_t i02 = div_mod.y; const int64_t i03 = div_mod.x; ++ const int64_t i12 = fastmodulo((uint32_t)i03, ne12_fd); ++ const int64_t i11 = fastmodulo((uint32_t)i02, ne11_fd); ++ const int64_t dst_row = *(src1 + i01*s10 + i11*s11 + i12*s12); ++ const float * grp_src = src0 + i01*s01 + i02*s02 + i03*s03 + i00; ++ block_turbo3_tcq * dst_blk = (block_turbo3_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3) ++ + (i00 / QK_TURBO3_TCQ); ++ ++ // Shared memory layout (~5KB, was ~35KB before global bt optimization): ++ // x[128] : rotated+normalized input (also reused as outputs[] after Viterbi) ++ // cost[512] : path costs buffer A (also reused for reductions) ++ // cost_b[512]: path costs buffer B (double-buffering eliminates 2/3 of syncs) ++ // Backtrace: one predecessor byte for each of the 64 low-state groups per ++ // step. The predecessor is independent of the output bits in sid[8:6], so ++ // storing 128×64 bytes is equivalent to the older 128×512 layout. ++ extern __shared__ uint8_t bt_shared[]; ++ __shared__ acc_t x[128]; ++ __shared__ acc_t cost[512]; ++ __shared__ acc_t cost_b[512]; ++ __shared__ int warp_min_idx[16]; ++ __shared__ acc_t warp_min_cost[16]; ++ __shared__ acc_t pred_min_cost[64]; ++ __shared__ int shared_initial_state; ++ ++ if (sid < 128) x[sid] = grp_src[sid]; ++ __syncthreads(); ++ ++ __syncthreads(); ++ ++ // Norm reduction ++ cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f; ++ __syncthreads(); ++ for (int stride = 256; stride >= 32; stride >>= 1) { ++ if (sid < stride) cost[sid] += cost[sid + stride]; ++ __syncthreads(); ++ } ++ if (sid < 32) { ++ acc_t v = cost[sid]; ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); ++ if (sid == 0) cost[0] = v; ++ } ++ __syncthreads(); ++ acc_t grp_norm = tcq_acc_sqrt(cost[0]); ++ acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f; ++ ++ if (sid < 128) x[sid] *= inv_norm; ++ __syncthreads(); ++ ++ // FWHT. The first five stages are contained within each 32-lane warp, so ++ // use warp shuffles and only synchronize for the two cross-warp stages. ++ if (sid < 128) { ++ acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid]; ++ const int lane = sid & 31; ++#pragma unroll ++ for (int h = 1; h < 32; h <<= 1) { ++ const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h); ++ v = (lane & h) ? (other - v) : (v + other); ++ } ++ x[sid] = v; ++ } ++ __syncthreads(); ++ if (sid < 64) { ++ const int j = ((sid >> 5) << 6) + (sid & 31); ++ acc_t a = x[j], b = x[j + 32]; ++ x[j] = a + b; x[j + 32] = a - b; ++ } ++ __syncthreads(); ++ if (sid < 64) { ++ acc_t a = x[sid], b = x[sid + 64]; ++ x[sid] = a + b; x[sid + 64] = a - b; ++ } ++ __syncthreads(); ++ constexpr float inv_sqrt_128 = 0.08838834764831845f; ++ if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid]; ++ __syncthreads(); ++ ++ // opencoti-hook: TCQ activation-dump (F-D, #515). x[0..127] is now the post-FWHT unit-norm whitened vec — ++ // exactly the xt the trellis quantizes against d_turbo3_tcq_codebook below, and x stays read-only through the ++ // forward pass. Stage it for the F-E codebook retrain when enabled; nullptr (default) is a uniform no-op. ++ // group == blockIdx.x < ne_total_groups, so [group*128 + sid] is in-bounds of the (ne_total_groups*128) buffer. ++ if (d_tcq_dump_buf != nullptr && sid < 128) { ++ d_tcq_dump_buf[group * 128 + sid] = (float) x[sid]; ++ } ++ ++ if (sid == 0) cost[0] = grp_norm; ++ __syncthreads(); ++ ++ acc_t saved_norm = cost[0]; ++ ++ // Viterbi forward pass: double-buffered cost (1 sync/step, was 3) ++ uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64); ++ cost[sid] = 0.0f; ++ __syncthreads(); ++ ++ for (int t = 0; t < 128; t++) { ++ // Double-buffer: even steps read cost/write cost_b, odd steps read cost_b/write cost ++ acc_t * cost_rd = (t & 1) ? cost_b : cost; ++ acc_t * cost_wr = (t & 1) ? cost : cost_b; ++ ++ acc_t xt = x[t]; ++ ++ // Right-shift trellis: ns = (prev >> 3) | (out << 6). The best ++ // predecessor depends only on sid's low 6 bits, so compute those 64 ++ // minima once instead of repeating the same 8-way scan for each out. ++ if (sid < 64) { ++ const int base_prev = sid << 3; ++ acc_t best = cost_rd[base_prev]; ++ int best_p = 0; ++#pragma unroll ++ for (int p = 1; p < 8; p++) { ++ acc_t c = cost_rd[base_prev | p]; ++ if (c < best) { ++ best = c; ++ best_p = p; ++ } ++ } ++ pred_min_cost[sid] = best; ++ bt[t * 64 + sid] = (uint8_t) best_p; ++ } ++ __syncthreads(); ++ ++ const int pred_idx = sid & 0x3F; ++ acc_t dist = xt - d_turbo3_tcq_codebook[sid]; ++ dist = dist * dist; ++ ++ cost_wr[sid] = pred_min_cost[pred_idx] + dist; ++ __syncthreads(); ++ } ++ // After 128 steps (even count): final costs are in cost[] (step 127 writes to cost) ++ ++ // Warp argmin over 512 costs ++ { ++ acc_t my_cost = cost[sid]; ++ int my_idx = sid; ++ #pragma unroll ++ for (int offset = 16; offset > 0; offset >>= 1) { ++ acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset); ++ int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset); ++ if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; } ++ } ++ if (sid % 32 == 0) { ++ warp_min_cost[sid / 32] = my_cost; ++ warp_min_idx[sid / 32] = my_idx; ++ } ++ } ++ __syncthreads(); ++ if (sid < 32) { ++ acc_t best = (sid < 16) ? warp_min_cost[sid] : 3.4028234663852886e38f; ++ int best_idx = (sid < 16) ? warp_min_idx[sid] : 0; ++#pragma unroll ++ for (int offset = 16; offset > 0; offset >>= 1) { ++ acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset); ++ int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset); ++ if (other_cost < best) { ++ best = other_cost; ++ best_idx = other_idx; ++ } ++ } ++ if (sid == 0) { ++ shared_initial_state = best_idx; // temporarily: best final state (becomes initial after backtrack) ++ } ++ } ++ __syncthreads(); ++ ++ // Save x[] to global buffer before backtrack overwrites it ++ ++ // Backtrack (inherently sequential, reads global bt) ++ uint8_t * outputs = (uint8_t *)x; ++ if (sid == 0) { ++ int state = shared_initial_state; ++ for (int t = 127; t >= 0; t--) { ++ outputs[t] = (uint8_t)(state >> 6); ++ int p = bt[t * 64 + (state & 0x3F)]; ++ state = ((state & 0x3F) << 3) | p; ++ } ++ shared_initial_state = state; ++ } ++ __syncthreads(); ++ ++ // Save output symbols to global buffer ++ ++ // Parallel recon norm: t>=2 can compute state directly from 3 outputs (3 shifts of 3 = 9 bits) ++ acc_t my_recon_sq = 0.0f; ++ if (sid < 128) { ++ int cur_state; ++ if (sid < 2) { ++ cur_state = shared_initial_state; ++ for (int t = 0; t <= sid; t++) ++ cur_state = (cur_state >> 3) | (((int)outputs[t]) << 6); ++ } else { ++ cur_state = ((int)outputs[sid - 2] & 0x7) ++ | (((int)outputs[sid - 1] & 0x7) << 3) ++ | (((int)outputs[sid] & 0x7) << 6); ++ } ++ acc_t c = d_turbo3_tcq_codebook[cur_state]; ++ my_recon_sq = c * c; ++ } ++ cost[sid] = my_recon_sq; ++ __syncthreads(); ++ for (int stride = 256; stride >= 32; stride >>= 1) { ++ if (sid < stride) cost[sid] += cost[sid + stride]; ++ __syncthreads(); ++ } ++ if (sid < 32) { ++ acc_t v = cost[sid]; ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); ++ if (sid == 0) cost[0] = v; ++ } ++ __syncthreads(); ++ acc_t recon_norm = tcq_acc_sqrt(cost[0]); ++ acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm; ++ corrected_norm *= innerq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v; ++ ++ // Parallel bitpack: qs stores 6 initial-state bits followed by 128 3-bit ++ // output symbols. Each byte is independent, so avoid the old serial OR loop. ++ if (sid < 49) { ++ const int init_bits = (shared_initial_state >> 3) & 0x3F; ++ uint8_t packed = 0; ++#pragma unroll ++ for (int bit = 0; bit < 8; bit++) { ++ const int pos = sid * 8 + bit; ++ int v = 0; ++ if (pos < 6) { ++ v = (init_bits >> pos) & 1; ++ } else { ++ const int sym_bit_pos = pos - 6; ++ const int sym_idx = sym_bit_pos / 3; ++ if (sym_idx < 128) { ++ v = (outputs[sym_idx] >> (sym_bit_pos % 3)) & 1; ++ } ++ } ++ packed |= (uint8_t)(v << bit); ++ } ++ dst_blk->qs[sid] = packed; ++ } ++ if (sid == 0) { ++ dst_blk->norm = __float2half((float)corrected_norm); ++ } ++} ++ ++// TCQ GET_ROWS dequantize (for non-FA paths) ++#define QR_TURBO3_TCQ 2 ++static __device__ __forceinline__ ++void dequantize_turbo3_tcq(const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo3_tcq * blk = (const block_turbo3_tcq *)vx + ib; ++ const float norm = __half2float(blk->norm); ++ ++ // Decode element iqs ++ { ++ const int t = iqs; ++ const int bit_pos = t * 3; ++ const int byte_idx = bit_pos / 8; ++ const int bit_off = bit_pos % 8; ++ const uint16_t raw = (uint16_t)blk->qs[byte_idx] | ((uint16_t)blk->qs[byte_idx + 1] << 8); ++ const int state = (raw >> bit_off) & 0x1FF; ++ v.x = d_turbo3_tcq_codebook[state] * norm; ++ } ++ // Decode element iqs + 64 (stride = half block size) ++ { ++ const int t = iqs + 64; ++ const int bit_pos = t * 3; ++ const int byte_idx = bit_pos / 8; ++ const int bit_off = bit_pos % 8; ++ const uint16_t raw = (uint16_t)blk->qs[byte_idx] | ((uint16_t)blk->qs[byte_idx + 1] << 8); ++ const int state = (raw >> bit_off) & 0x1FF; ++ v.y = d_turbo3_tcq_codebook[state] * norm; ++ } ++} ++ ++// ===================================================================================== ++// TURBO2_TCQ: 2-bit Trellis-Coded Quantization (k=2, L=8, 256 states, free initial state) ++// ===================================================================================== ++ ++// 2-bit TCQ codebook (product_mono/iter090, 256-state bitshift trellis). If you copy these, credit spiritbuun! ++// CUDA GLA product-aware training, 100 iters on Qwen3.5-27B FWHT-rotated KV activations. Decode: state_t = read_8_bits(qs, t*2) ++static __constant__ float d_turbo2_tcq_codebook[256] = { ++ -0.18030643f, -0.11009848f, -0.04742626f, +0.02894132f, -0.10523465f, -0.031312924f, +0.031491395f, +0.12263535f, ++ -0.15660362f, -0.055477407f, +0.0046675834f, +0.06166081f, -0.07506216f, -0.016963918f, +0.043737844f, +0.116496615f, ++ -0.08632783f, -0.022493735f, +0.041032985f, +0.10660284f, -0.06274858f, -0.0036939639f, +0.02095157f, +0.07539709f, ++ -0.09802641f, -0.008419088f, +0.059072323f, +0.17311879f, -0.093109086f, -0.02654333f, +0.014827672f, +0.07793592f, ++ -0.031235758f, +0.01271591f, +0.08752262f, +0.17246453f, -0.14595252f, -0.07227624f, +0.013628688f, +0.08131674f, ++ -0.036909282f, +0.0018896917f, +0.05209119f, +0.12407892f, -0.13689458f, -0.06054520f, +0.0064648795f, +0.07551241f, ++ -0.18980840f, -0.110128626f, -0.046503957f, +0.026387159f, -0.034967307f, +0.04810357f, +0.072072044f, +0.14355458f, ++ -0.10182410f, -0.02907887f, +0.014033012f, +0.083419636f, -0.056140676f, +0.008405868f, +0.066070884f, +0.14037225f, ++ -0.117427245f, -0.047159385f, +0.016928354f, +0.08142885f, -0.029359628f, +0.045608785f, +0.10559447f, +0.20061271f, ++ -0.040425077f, +0.029068163f, +0.08408973f, +0.13628258f, -0.16633821f, -0.10711727f, -0.04196669f, +0.027895834f, ++ -0.0054065837f, +0.058898676f, +0.12688550f, +0.18268861f, -0.16287325f, -0.11218357f, -0.07165227f, -0.009524379f, ++ -0.24026902f, -0.073219374f, -0.0005165726f, +0.05959821f, -0.05532953f, +0.027044486f, +0.09425678f, +0.15356481f, ++ -0.14381111f, -0.10563502f, -0.037867088f, +0.023611993f, -0.03624307f, +0.049588434f, +0.12192037f, +0.23462485f, ++ -0.14990251f, -0.09659304f, -0.05886742f, +0.014878461f, -0.009889551f, +0.06910514f, +0.12120181f, +0.22596690f, ++ -0.08290075f, -0.009009629f, +0.066151775f, +0.12188313f, -0.11591514f, -0.06952189f, -0.031633306f, +0.023740824f, ++ -0.20510401f, -0.103369795f, +0.09148037f, +0.17268716f, -0.16597997f, -0.09207068f, -0.032810967f, +0.024847647f, ++ -0.02487482f, +0.049298953f, +0.09624215f, +0.14217524f, -0.18418685f, -0.10147012f, -0.05841265f, +0.008057022f, ++ -0.14269894f, -0.092456274f, -0.026881337f, +0.049792137f, -0.019881032f, +0.030333601f, +0.09736802f, +0.17764080f, ++ -0.19579841f, -0.114739306f, -0.026823774f, +0.07466014f, -0.09001050f, -0.041468445f, +0.028473806f, +0.08870695f, ++ -0.019396419f, +0.042828932f, +0.10885327f, +0.13335012f, -0.15005013f, -0.074581385f, -0.028608415f, +0.03848942f, ++ -0.09687270f, -0.057059396f, +0.0077843578f, +0.06302297f, -0.23247094f, -0.14509225f, -0.032651436f, +0.027010715f, ++ -0.047595482f, +0.06280303f, +0.114691675f, +0.17124057f, -0.21092793f, -0.13704823f, -0.07340412f, +0.0039013291f, ++ -0.062834196f, +0.012601906f, +0.012601906f, +0.08721347f, -0.13256435f, -0.024173854f, +0.07723171f, +0.14801070f, ++ -0.06471605f, -0.0017903054f, -0.0017903054f, +0.058302354f, -0.09731802f, -0.03400696f, +0.02762442f, +0.08986137f, ++ -0.08288722f, -0.019051429f, +0.045709886f, +0.15211061f, -0.09507891f, -0.015612489f, +0.025347246f, +0.087257534f, ++ -0.066236064f, -0.0047936034f, +0.06386274f, +0.15401669f, -0.105809286f, -0.051802177f, +0.01073050f, +0.08292137f, ++ -0.11884470f, -0.04404144f, +0.02550729f, +0.02550729f, -0.01731189f, +0.062161792f, +0.12127554f, +0.21981733f, ++ -0.17066145f, -0.11660990f, -0.049425896f, +0.021293938f, -0.04711412f, +0.026577346f, +0.055197213f, +0.12541275f, ++ -0.028268812f, +0.015206398f, +0.09002519f, +0.12699963f, -0.10059831f, -0.026676945f, +0.059903253f, +0.13054545f, ++ -0.09582803f, -0.033371232f, +0.010346129f, +0.066766635f, -0.09964944f, -0.028686784f, +0.021184925f, +0.09120017f, ++ -0.16957201f, -0.07594450f, +0.04172865f, +0.18313301f, -0.051526368f, +0.011877304f, +0.011877304f, +0.07956263f, ++ -0.13432936f, -0.05269006f, +0.03536416f, +0.117640756f, -0.022776067f, +0.042032316f, +0.10472976f, +0.18042557f ++}; ++ ++// 2-bit TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis (k=2, L=8) ++// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384) ++template ++static __global__ void __launch_bounds__(256, 1) k_set_rows_turbo2_tcq( ++ const float * __restrict__ src0, const idx_t * __restrict__ src1, ++ block_turbo2_tcq * __restrict__ dst, const int64_t ne_total_groups, ++ uint8_t * __restrict__ bt_buf, ++ const int use_shared_bt, ++ const int64_t ne00, const int64_t ne01, const int64_t ne02, ++ const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, ++ const int64_t s01, const int64_t s02, const int64_t s03, ++ const int64_t s10, const int64_t s11, const int64_t s12, ++ const int iq_is_k, ++ const int64_t s1, const int64_t s2, const int64_t s3, ++ const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd, ++ const uint3 ne11_fd, const uint3 ne12_fd) { ++ ++ const int grp = blockIdx.x; ++ if (grp >= ne_total_groups) return; ++ const int sid = threadIdx.x; // 0..255 = trellis state ++ ++ // Compute source and destination pointers (all threads, used by thread 0) ++ const int64_t i_base = int64_t(grp) * QK_TURBO2_TCQ; ++ uint32_t tmp = (uint32_t)i_base; uint2 div_mod; ++ div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x; ++ div_mod = fast_div_modulo(tmp, ne01_fd); const int64_t i01 = div_mod.y; tmp = div_mod.x; ++ div_mod = fast_div_modulo(tmp, ne02_fd); const int64_t i02 = div_mod.y; const int64_t i03 = div_mod.x; ++ const int64_t i12 = fastmodulo((uint32_t)i03, ne12_fd); ++ const int64_t i11 = fastmodulo((uint32_t)i02, ne11_fd); ++ const int64_t dst_row = *(src1 + i01*s10 + i11*s11 + i12*s12); ++ const float * grp_src = src0 + i01*s01 + i02*s02 + i03*s03 + i00; ++ block_turbo2_tcq * dst_blk = (block_turbo2_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3) ++ + (i00 / QK_TURBO2_TCQ); ++ ++ // Backtrace: one predecessor byte per 64 low-state groups per step. ++ // The predecessor depends only on sid's low 6 bits (same as turbo3_tcq). ++ extern __shared__ uint8_t bt_shared[]; ++ __shared__ acc_t x[128]; ++ __shared__ acc_t cost[256]; ++ __shared__ acc_t cost_b[256]; // double-buffering for Viterbi ++ __shared__ int warp_min_idx[8]; ++ __shared__ acc_t warp_min_cost[8]; ++ __shared__ acc_t pred_min_cost[64]; ++ __shared__ int shared_initial_state; ++ ++ if (sid < 128) x[sid] = grp_src[sid]; ++ __syncthreads(); ++ ++ __syncthreads(); ++ ++ // Norm reduction ++ cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f; ++ __syncthreads(); ++ for (int stride = 128; stride >= 32; stride >>= 1) { ++ if (sid < stride) cost[sid] += cost[sid + stride]; ++ __syncthreads(); ++ } ++ if (sid < 32) { ++ acc_t v = cost[sid]; ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); ++ if (sid == 0) cost[0] = v; ++ } ++ __syncthreads(); ++ acc_t grp_norm = tcq_acc_sqrt(cost[0]); ++ acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f; ++ ++ if (sid < 128) x[sid] *= inv_norm; ++ __syncthreads(); ++ ++ // FWHT. The first five stages use warp shuffles, the two cross-warp stages ++ // use shared memory (same approach as turbo3_tcq). ++ if (sid < 128) { ++ acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid]; ++ const int lane = sid & 31; ++#pragma unroll ++ for (int h = 1; h < 32; h <<= 1) { ++ const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h); ++ v = (lane & h) ? (other - v) : (v + other); ++ } ++ x[sid] = v; ++ } ++ __syncthreads(); ++ if (sid < 64) { ++ const int j = ((sid >> 5) << 6) + (sid & 31); ++ acc_t a = x[j], b = x[j + 32]; ++ x[j] = a + b; x[j + 32] = a - b; ++ } ++ __syncthreads(); ++ if (sid < 64) { ++ acc_t a = x[sid], b = x[sid + 64]; ++ x[sid] = a + b; x[sid + 64] = a - b; ++ } ++ __syncthreads(); ++ constexpr float inv_sqrt_128 = 0.08838834764831845f; ++ if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid]; ++ __syncthreads(); ++ ++ if (sid == 0) cost[0] = grp_norm; ++ __syncthreads(); ++ ++ acc_t saved_norm = cost[0]; ++ ++ // Viterbi forward pass: double-buffered cost (1 sync/step, was 3) ++ uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64); ++ cost[sid] = 0.0f; ++ __syncthreads(); ++ ++ for (int t = 0; t < 128; t++) { ++ acc_t * cost_rd = (t & 1) ? cost_b : cost; ++ acc_t * cost_wr = (t & 1) ? cost : cost_b; ++ ++ acc_t xt = x[t]; ++ ++ // Right-shift trellis (k=2, L=8): ns = (prev >> 2) | (out << 6). ++ // The best predecessor depends only on sid's low 6 bits, so compute ++ // those 64 minima once instead of repeating the same 4-way scan per state. ++ if (sid < 64) { ++ const int base_prev = sid << 2; ++ acc_t best = cost_rd[base_prev]; ++ int best_p = 0; ++#pragma unroll ++ for (int p = 1; p < 4; p++) { ++ acc_t c = cost_rd[base_prev | p]; ++ if (c < best) { ++ best = c; ++ best_p = p; ++ } ++ } ++ pred_min_cost[sid] = best; ++ bt[t * 64 + sid] = (uint8_t) best_p; ++ } ++ __syncthreads(); ++ ++ const int pred_idx = sid & 0x3F; ++ acc_t dist = xt - d_turbo2_tcq_codebook[sid]; ++ dist = dist * dist; ++ ++ cost_wr[sid] = pred_min_cost[pred_idx] + dist; ++ __syncthreads(); ++ } ++ // After 128 steps (even count): final costs in cost[] ++ ++ // Warp argmin over 256 costs ++ { ++ acc_t my_cost = cost[sid]; ++ int my_idx = sid; ++ #pragma unroll ++ for (int offset = 16; offset > 0; offset >>= 1) { ++ acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset); ++ int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset); ++ if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; } ++ } ++ if (sid % 32 == 0) { ++ warp_min_cost[sid / 32] = my_cost; ++ warp_min_idx[sid / 32] = my_idx; ++ } ++ } ++ __syncthreads(); ++ if (sid < 32) { ++ acc_t best = (sid < 8) ? warp_min_cost[sid] : 3.4028234663852886e38f; ++ int best_idx = (sid < 8) ? warp_min_idx[sid] : 0; ++#pragma unroll ++ for (int offset = 16; offset > 0; offset >>= 1) { ++ acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset); ++ int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset); ++ if (other_cost < best) { ++ best = other_cost; ++ best_idx = other_idx; ++ } ++ } ++ if (sid == 0) { ++ shared_initial_state = best_idx; ++ } ++ } ++ __syncthreads(); ++ ++ // Save x[] to global buffer before backtrack overwrites it ++ ++ // Backtrack (inherently sequential, reads compressed bt) ++ uint8_t * outputs = (uint8_t *)x; ++ if (sid == 0) { ++ int state = shared_initial_state; ++ for (int t = 127; t >= 0; t--) { ++ outputs[t] = (uint8_t)(state >> 6); ++ int p = bt[t * 64 + (state & 0x3F)]; ++ state = ((state & 0x3F) << 2) | p; ++ } ++ shared_initial_state = state; ++ } ++ __syncthreads(); ++ ++ // Save output symbols to global buffer ++ ++ // Parallel recon norm: t>=3 can compute state directly from 4 outputs (4 shifts of 2 = 8 bits) ++ acc_t my_recon_sq = 0.0f; ++ if (sid < 128) { ++ int cur_state; ++ if (sid < 3) { ++ cur_state = shared_initial_state; ++ for (int t = 0; t <= sid; t++) ++ cur_state = (cur_state >> 2) | (((int)outputs[t]) << 6); ++ } else { ++ cur_state = ((int)outputs[sid - 3] & 0x3) ++ | (((int)outputs[sid - 2] & 0x3) << 2) ++ | (((int)outputs[sid - 1] & 0x3) << 4) ++ | (((int)outputs[sid] & 0x3) << 6); ++ } ++ acc_t c = d_turbo2_tcq_codebook[cur_state]; ++ my_recon_sq = c * c; ++ } ++ cost[sid] = my_recon_sq; ++ __syncthreads(); ++ for (int stride = 128; stride >= 32; stride >>= 1) { ++ if (sid < stride) cost[sid] += cost[sid + stride]; ++ __syncthreads(); ++ } ++ if (sid < 32) { ++ acc_t v = cost[sid]; ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); ++ v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); ++ if (sid == 0) cost[0] = v; ++ } ++ __syncthreads(); ++ acc_t recon_norm = tcq_acc_sqrt(cost[0]); ++ acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm; ++ corrected_norm *= iq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v; ++ ++ // Parallel bitpack: qs stores 6 initial-state bits followed by 128 2-bit ++ // output symbols. Each byte is independent (2-bit symbols never cross byte ++ // boundaries after the 6-bit prefix), so avoid the serial OR loop. ++ if (sid < 33) { ++ const int init_bits = (shared_initial_state >> 2) & 0x3F; ++ uint8_t packed = 0; ++#pragma unroll ++ for (int bit = 0; bit < 8; bit++) { ++ const int pos = sid * 8 + bit; ++ int v = 0; ++ if (pos < 6) { ++ v = (init_bits >> pos) & 1; ++ } else { ++ const int sym_bit_pos = pos - 6; ++ const int sym_idx = sym_bit_pos / 2; ++ if (sym_idx < 128) { ++ v = (outputs[sym_idx] >> (sym_bit_pos % 2)) & 1; ++ } ++ } ++ packed |= (uint8_t)(v << bit); ++ } ++ dst_blk->qs[sid] = packed; ++ } ++ if (sid == 0) { ++ dst_blk->norm = __float2half((float)corrected_norm); ++ } ++} ++ ++// 2-bit TCQ GET_ROWS dequantize ++#define QR_TURBO2_TCQ 2 ++static __device__ __forceinline__ ++void dequantize_turbo2_tcq(const void * vx, const int64_t ib, const int iqs, float2 & v) { ++ const block_turbo2_tcq * blk = (const block_turbo2_tcq *)vx + ib; ++ const float norm = __half2float(blk->norm); ++ ++ // Decode element iqs: read 8-bit state via sliding window ++ { ++ const int t = iqs; ++ const int bit_pos = t * 2; ++ const int byte_idx = bit_pos / 8; ++ const int bit_off = bit_pos % 8; ++ const uint16_t raw = (uint16_t)blk->qs[byte_idx] | ((uint16_t)blk->qs[byte_idx + 1] << 8); ++ const int state = (raw >> bit_off) & 0xFF; ++ v.x = d_turbo2_tcq_codebook[state] * norm; ++ } ++ // Decode element iqs + 64 ++ { ++ const int t = iqs + 64; ++ const int bit_pos = t * 2; ++ const int byte_idx = bit_pos / 8; ++ const int bit_off = bit_pos % 8; ++ const uint16_t raw = (uint16_t)blk->qs[byte_idx] | ((uint16_t)blk->qs[byte_idx + 1] << 8); ++ const int state = (raw >> bit_off) & 0xFF; ++ v.y = d_turbo2_tcq_codebook[state] * norm; ++ } ++} +diff --git a/llama.cpp/ggml/src/ggml-quants.c b/llama.cpp/ggml/src/ggml-quants.c +index 15d231f70..3ac0f340a 100644 +--- a/llama.cpp/ggml/src/ggml-quants.c ++++ b/llama.cpp/ggml/src/ggml-quants.c +@@ -190,6 +190,57 @@ void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_REST + } + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — see docs/features/poly_kv.md. ++// Signed 6-bit (levels offset by 32). qs[16] = low 4 bits of all 32 values; ++// qh[8] = high 2 bits, packed 4 values/byte. Ported from ik_llama ggml-quants.c:853. ++void quantize_row_q6_0_ref(const float * GGML_RESTRICT x, block_q6_0 * GGML_RESTRICT y, int64_t k) { ++ static const int qk = QK6_0; ++ ++ assert(k % qk == 0); ++ ++ const int nb = k / qk; ++ ++ for (int i = 0; i < nb; i++) { ++ float amax = 0.0f; // absolute max ++ float max = 0.0f; ++ ++ for (int j = 0; j < qk; j++) { ++ const float v = x[i*qk + j]; ++ if (amax < fabsf(v)) { ++ amax = fabsf(v); ++ max = v; ++ } ++ } ++ ++ const float d = max / -32; ++ const float id = d ? 1.0f/d : 0.0f; ++ ++ memset(y[i].qh, 0, qk/4); ++ ++ float sumqx = 0, sumq2 = 0; ++ for (int j = 0; j < qk/2; ++j) { ++ const float x0 = x[i*qk + 0 + j]*id; ++ const float x1 = x[i*qk + qk/2 + j]*id; ++ const float w0 = x0*x0; ++ const float w1 = x1*x1; ++ ++ const uint8_t xi0 = MIN(63, (int8_t)(x0 + 32.5f)); ++ const uint8_t xi1 = MIN(63, (int8_t)(x1 + 32.5f)); ++ ++ y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); ++ ++ const uint8_t h = (xi0 >> 4) | ((xi1 >> 4) << 2); ++ y[i].qh[j%(qk/4)] |= (h << 4*(j/(qk/4))); ++ ++ const float q0 = (float)xi0 - 32.f; ++ const float q1 = (float)xi1 - 32.f; ++ sumqx += w0*x[i*qk + j]*q0 + w1*x[i*qk + qk/2 + j]*q1; ++ sumq2 += w0*q0*q0 + w1*q1*q1; ++ } ++ y[i].d = sumq2 > 0 ? GGML_FP32_TO_FP16(sumqx/sumq2) : GGML_FP32_TO_FP16(d); ++ } ++} ++ + void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k) { + const int qk = QK5_1; + +@@ -465,6 +516,29 @@ void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRI + } + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — ported from ik_llama ggml-quants.c:1675. ++void dequantize_row_q6_0(const block_q6_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ static const int qk = QK6_0; ++ ++ assert(k % qk == 0); ++ ++ const int nb = k / qk; ++ ++ for (int i = 0; i < nb; i++) { ++ const float d = GGML_FP16_TO_FP32(x[i].d); ++ ++ for (int j = 0; j < qk/2; ++j) { ++ const uint8_t h = x[i].qh[j%(qk/4)] >> 4*(j/(qk/4)); ++ ++ const int32_t x0 = ((x[i].qs[j] & 0x0F) | ((h << 4) & 0x30)) - 32; ++ const int32_t x1 = ((x[i].qs[j] >> 4) | ((h << 2) & 0x30)) - 32; ++ ++ y[i*qk + j + 0 ] = x0*d; ++ y[i*qk + j + qk/2] = x1*d; ++ } ++ } ++} ++ + void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK5_1; + +@@ -2167,6 +2241,58 @@ size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + return nrow * row_size; + } + ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — KV path (quant_weights==NULL) routes to the ++// byte-exact ref; imatrix path mirrors ik_llama ggml-quants.c:3613 for GGUF-weight completeness. ++static void quantize_row_q6_0_impl(const float * GGML_RESTRICT x, block_q6_0 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) { ++ static_assert(QK6_0 == 32, "QK6_0 must be 32"); ++ ++ if (!quant_weights) { ++ quantize_row_q6_0_ref(x, y, n_per_row); ++ return; ++ } ++ ++ float weight[QK6_0]; ++ int8_t L[QK6_0]; ++ ++ float sum_x2 = 0; ++ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; ++ float sigma2 = sum_x2/n_per_row; ++ ++ const int64_t nb = n_per_row/QK6_0; ++ for (int ib = 0; ib < nb; ++ib) { ++ const float * xb = x + QK6_0 * ib; ++ const float * qw = quant_weights + QK6_0 * ib; ++ for (int j = 0; j < QK6_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); ++ float d = make_qx_quants(QK6_0, 32, xb, L, 1, weight); ++ y[ib].d = GGML_FP32_TO_FP16(d); ++ ++ memset(y[ib].qh, 0, QK6_0/4); ++ ++ for (int j = 0; j < 16; ++j) { ++ const uint8_t xi0 = L[j]; ++ const uint8_t xi1 = L[j+16]; ++ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); ++ const uint8_t h = (xi0 >> 4) | ((xi1 >> 4) << 2); ++ y[ib].qh[j%8] |= (h << 4*(j/8)); ++ } ++ } ++} ++ ++size_t quantize_q6_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { ++ if (!quant_weights) { ++ quantize_row_q6_0_ref(src, dst, (int64_t)nrow*n_per_row); ++ return nrow * ggml_row_size(GGML_TYPE_Q6_0, n_per_row); ++ } ++ size_t row_size = ggml_row_size(GGML_TYPE_Q6_0, n_per_row); ++ char * qrow = (char *)dst; ++ for (int64_t row = 0; row < nrow; ++row) { ++ quantize_row_q6_0_impl(src, (block_q6_0*)qrow, n_per_row, quant_weights); ++ src += n_per_row; ++ qrow += row_size; ++ } ++ return nrow * row_size; ++} ++ + static void quantize_row_q5_1_impl(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t n_per_row, const float * quant_weights) { + static_assert(QK5_1 == 32, "QK5_1 must be 32"); + +diff --git a/llama.cpp/ggml/src/ggml-quants.h b/llama.cpp/ggml/src/ggml-quants.h +index 75c6efa82..d77693e92 100644 +--- a/llama.cpp/ggml/src/ggml-quants.h ++++ b/llama.cpp/ggml/src/ggml-quants.h +@@ -21,6 +21,8 @@ GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * + GGML_API void quantize_row_q5_1_ref(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k); + GGML_API void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k); + GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k); ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++GGML_API void quantize_row_q6_0_ref(const float * GGML_RESTRICT x, block_q6_0 * GGML_RESTRICT y, int64_t k); + + GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); + GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k); +@@ -49,6 +51,8 @@ GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GG + GGML_API void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + //GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++GGML_API void dequantize_row_q6_0(const block_q6_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + + GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +@@ -98,6 +102,8 @@ GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTR + GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + GGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) ++GGML_API size_t quantize_q6_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + + GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +@@ -121,6 +127,14 @@ GGML_API size_t quantize_turbo2_0(const float * GGML_RESTRICT src, void * GGML_R + GGML_API void quantize_row_turbo8_0_ref(const float * GGML_RESTRICT x, block_turbo8_0 * GGML_RESTRICT y, int64_t k); + GGML_API void dequantize_row_turbo8_0(const block_turbo8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + GGML_API size_t quantize_turbo8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++// opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV (buun-aligned names). CPU stubs only; ++// real encode/decode live in CUDA (set-rows / FA-VEC). See ggml-turbo-quant.c. ++GGML_API void quantize_row_turbo3_tcq_ref(const float * GGML_RESTRICT x, block_turbo3_tcq * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo3_tcq(const block_turbo3_tcq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_turbo3_tcq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); ++GGML_API void quantize_row_turbo2_tcq_ref(const float * GGML_RESTRICT x, block_turbo2_tcq * GGML_RESTRICT y, int64_t k); ++GGML_API void dequantize_row_turbo2_tcq(const block_turbo2_tcq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); ++GGML_API size_t quantize_turbo2_tcq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + + // TQ3_1S: WHT-rotated 3-bit weight quantization (8-level Lloyd-Max) + GGML_API void quantize_row_tq3_1s_ref(const float * GGML_RESTRICT x, block_tq3_1s * GGML_RESTRICT y, int64_t k); +diff --git a/llama.cpp/ggml/src/ggml-turbo-quant.c b/llama.cpp/ggml/src/ggml-turbo-quant.c +index 38e15feca..156502191 100644 +--- a/llama.cpp/ggml/src/ggml-turbo-quant.c ++++ b/llama.cpp/ggml/src/ggml-turbo-quant.c +@@ -652,6 +652,91 @@ size_t quantize_turbo8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT d + return nrows * row_size; + } + ++/* ---------- TURBO3_TCQ / TURBO2_TCQ: Trellis-Coded Quantization (CPU stubs) ---------- ++ * opencoti-hook: TCQ (F5 M6 C5), ported verbatim from buun (spiritbuun/buun-llama-cpp). ++ * TCQ is GPU-only: the real encode (FWHT + Viterbi) is the CUDA SET_ROWS kernel and the real ++ * decode (O(1) sliding-window codebook lookup) is inlined in the FA-VEC kernel. These CPU paths ++ * are deliberate stubs — from_float writes only the corrected group L2 norm (so the ++ * dequant-on-lift / POSITION_WINDOW host-pinned tail keeps a consistent norm), to_float zero-fills. ++ */ ++void quantize_row_turbo3_tcq_ref(const float * GGML_RESTRICT x, block_turbo3_tcq * GGML_RESTRICT y, int64_t k) { ++ // Stub — CUDA kernel handles TCQ quantize (Viterbi). CPU path zeros out. ++ assert(k % QK_TURBO3_TCQ == 0); ++ const int nb = k / QK_TURBO3_TCQ; ++ for (int i = 0; i < nb; i++) { ++ float norm = 0.0f; ++ for (int j = 0; j < QK_TURBO3_TCQ; j++) norm += x[i*QK_TURBO3_TCQ + j] * x[i*QK_TURBO3_TCQ + j]; ++ y[i].norm = GGML_FP32_TO_FP16(sqrtf(norm)); ++ memset(y[i].qs, 0, 49); ++ } ++} ++ ++void dequantize_row_turbo3_tcq(const block_turbo3_tcq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ GGML_UNUSED(x); ++ assert(k % QK_TURBO3_TCQ == 0); ++ const int nb = k / QK_TURBO3_TCQ; ++ for (int block = 0; block < nb; block++) { ++ for (int j = 0; j < QK_TURBO3_TCQ; j++) { ++ y[block * QK_TURBO3_TCQ + j] = 0.0f; ++ } ++ } ++} ++ ++size_t quantize_turbo3_tcq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO3_TCQ == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO3_TCQ) * sizeof(block_turbo3_tcq); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo3_tcq_ref( ++ src + row * n_per_row, ++ (block_turbo3_tcq *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ ++void quantize_row_turbo2_tcq_ref(const float * GGML_RESTRICT x, block_turbo2_tcq * GGML_RESTRICT y, int64_t k) { ++ // Stub — CUDA kernel handles TCQ quantize (Viterbi). CPU path zeros out. ++ assert(k % QK_TURBO2_TCQ == 0); ++ const int nb = k / QK_TURBO2_TCQ; ++ for (int i = 0; i < nb; i++) { ++ float norm = 0.0f; ++ for (int j = 0; j < QK_TURBO2_TCQ; j++) norm += x[i*QK_TURBO2_TCQ + j] * x[i*QK_TURBO2_TCQ + j]; ++ y[i].norm = GGML_FP32_TO_FP16(sqrtf(norm)); ++ memset(y[i].qs, 0, 33); ++ } ++} ++ ++void dequantize_row_turbo2_tcq(const block_turbo2_tcq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { ++ GGML_UNUSED(x); ++ assert(k % QK_TURBO2_TCQ == 0); ++ const int nb = k / QK_TURBO2_TCQ; ++ for (int block = 0; block < nb; block++) { ++ for (int j = 0; j < QK_TURBO2_TCQ; j++) { ++ y[block * QK_TURBO2_TCQ + j] = 0.0f; ++ } ++ } ++} ++ ++size_t quantize_turbo2_tcq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, ++ int64_t nrows, int64_t n_per_row, const float * imatrix) { ++ GGML_UNUSED(imatrix); ++ assert(n_per_row % QK_TURBO2_TCQ == 0); ++ ++ size_t row_size = (n_per_row / QK_TURBO2_TCQ) * sizeof(block_turbo2_tcq); ++ for (int64_t row = 0; row < nrows; row++) { ++ quantize_row_turbo2_tcq_ref( ++ src + row * n_per_row, ++ (block_turbo2_tcq *)((char *)dst + row * row_size), ++ n_per_row ++ ); ++ } ++ return nrows * row_size; ++} ++ + /* ---------- TURBO4_0: 3-bit PolarQuant + 1-bit QJL ---------- */ + + void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k) { +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +index d19e81f9b..2defaaefd 100644 +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -707,6 +707,32 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { + .to_float = (ggml_to_float_t) dequantize_row_turbo8_0, + .from_float_ref = (ggml_from_float_t) quantize_row_turbo8_0_ref, + }, ++ // opencoti-hook: TCQ (F5 M6 C5) — Trellis-Coded Quantization KV (buun-aligned names) ++ [GGML_TYPE_TURBO3_TCQ] = { ++ .type_name = "turbo3_tcq", ++ .blck_size = QK_TURBO3_TCQ, ++ .type_size = sizeof(block_turbo3_tcq), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo3_tcq, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo3_tcq_ref, ++ }, ++ [GGML_TYPE_TURBO2_TCQ] = { ++ .type_name = "turbo2_tcq", ++ .blck_size = QK_TURBO2_TCQ, ++ .type_size = sizeof(block_turbo2_tcq), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_turbo2_tcq, ++ .from_float_ref = (ggml_from_float_t) quantize_row_turbo2_tcq_ref, ++ }, ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — scalar 6-bit KV tier, mirrors q5_0. ++ [GGML_TYPE_Q6_0] = { ++ .type_name = "q6_0", ++ .blck_size = QK6_0, ++ .type_size = sizeof(block_q6_0), ++ .is_quantized = true, ++ .to_float = (ggml_to_float_t) dequantize_row_q6_0, ++ .from_float_ref = (ggml_from_float_t) quantize_row_q6_0_ref, ++ }, + [GGML_TYPE_TQ3_1S] = { + .type_name = "tq3_1s", + .blck_size = QK_TQ3_0, +@@ -8122,6 +8148,8 @@ size_t ggml_quantize_chunk( + case GGML_TYPE_Q4_1: result = quantize_q4_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q5_0: result = quantize_q5_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q5_1: result = quantize_q5_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ // opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — see docs/features/poly_kv.md. ++ case GGML_TYPE_Q6_0: result = quantize_q6_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; +@@ -8146,6 +8174,9 @@ size_t ggml_quantize_chunk( + case GGML_TYPE_TURBO4_0: result = quantize_turbo4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_TURBO2_0: result = quantize_turbo2_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_TURBO8_0: result = quantize_turbo8_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ // opencoti-hook: TCQ (F5 M6 C5) ++ case GGML_TYPE_TURBO3_TCQ: result = quantize_turbo3_tcq(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; ++ case GGML_TYPE_TURBO2_TCQ: result = quantize_turbo2_tcq(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_TQ3_1S: result = quantize_tq3_1s(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_TQ4_1S: result = quantize_tq4_1s(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_F16: +diff --git a/llama.cpp/ggml/src/iqk/iqk_common_extra.h b/llama.cpp/ggml/src/iqk/iqk_common_extra.h +index 969379271..85f006006 100644 +--- a/llama.cpp/ggml/src/iqk/iqk_common_extra.h ++++ b/llama.cpp/ggml/src/iqk/iqk_common_extra.h +@@ -14,13 +14,11 @@ typedef struct { + } block_q5_0_r4; + static_assert(sizeof(block_q5_0_r4) == 4*sizeof(ggml_half) + QK5_0*2 + QK5_0/2, "wrong q5_0_r4 block size/padding"); + +-#define QK6_0 32 +-typedef struct { +- ggml_half d; // delta +- uint8_t qh[QK6_0/4]; // 5+6-th bit of quants +- uint8_t qs[QK6_0/2]; // nibbles / quants +-} block_q6_0; +-static_assert(sizeof(block_q6_0) == sizeof(ggml_half) + QK6_0/2 + QK6_0/4, "wrong q6_0 block size/padding"); ++#define QK6_0 32 // kept: block_q6_0_r4 below uses it; identical to ggml-common.h's #define (harmless redef) ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port, F-A) — block_q6_0 now lives in the real ggml-common.h ++// (this delta's shim includes "../ggml-common.h" BEFORE this file), so the ik copy is removed. Two anonymous ++// -struct typedefs of the same name are DISTINCT types in C++ → "typedef redefinition with different types" ++// (bug-611). Re-sync per this header's contract: drop block types once the main ggml-common.h provides them. + + typedef struct { + ggml_half d[4]; // delta +diff --git a/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h b/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h +index 673828191..0b1d97353 100644 +--- a/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h ++++ b/llama.cpp/ggml/src/iqk/iqk_ggml_type_ext.h +@@ -30,9 +30,10 @@ + #ifndef GGML_TYPE_Q8_2_X4 + #define GGML_TYPE_Q8_2_X4 ((ggml_type)99) + #endif +-#ifndef GGML_TYPE_Q6_0 +-#define GGML_TYPE_Q6_0 ((ggml_type)133) +-#endif ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port, F-A) — GGML_TYPE_Q6_0 is now a NATIVE llamafile enum ++// value (=50). The old ik delta `#define GGML_TYPE_Q6_0 ((ggml_type)133)` is removed: #ifndef cannot see an ++// enum constant (only macros), so it would define 133 and SHADOW the real 50 → iqk and main ggml disagree on ++// q6_0's identity (bug-611). iqk TUs already see ggml.h's enum (they use Q4_0/Q8_0/etc.), so 50 resolves cleanly. + #ifndef GGML_TYPE_IQ1_BN + #define GGML_TYPE_IQ1_BN ((ggml_type)134) + #endif +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +index 78e0b43bc..b84ff8374 100644 +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -175,6 +175,35 @@ ggml_tensor * llm_graph_context::build_dca_rope(ggml_tensor * x, ggml_tensor * p + ext_factor, attn_factor, beta_fast, beta_slow); + } + ++// opencoti F5 dca #444 (DCA all-KV C1): dequant-on-lift a cached K/V tensor to f16 for the f16-only ++// fused DCA kernel (the cache itself stays quantized — that is the memory win; we only materialize a ++// transient f16 copy for the kernel). The CUDA CPY backend has DIRECT ->f16 casts for f32/bf16/turbo ++// (turbo via the #385 InnerQ-aware kernel), and scalar-quant->f32 casts (q8_0/q4_0/q4_1/q5_0/q5_1), ++// but NO direct scalar-quant->f16 — that pair fails CPY supports_op and the scheduler CPU-spills it ++// (whole-cache dequant on the CPU + H2D copy per layer per token = a decode cliff). So route scalar- ++// quantized caches through f32 first: both legs are GPU CPY kernels, and the f32 intermediate is ++// NUMERICALLY IDENTICAL to a one-pass dequant (d*q is computed in f32 then rounded to f16 either way), ++// so this is byte-exact to the eventual single-pass quant->f16 kernel. f16 returns unchanged (byte- ++// identical to the prior assert path). The single-pass direct quant->f16 CPY kernels are the C-series ++// perf follow-on (built + restamped at #448); this helper keeps C1 host-only and immediately gateable. ++static ggml_tensor * dca_lift_to_f16(ggml_context * ctx0, ggml_tensor * x) { ++ if (x->type == GGML_TYPE_F16) { ++ return x; ++ } ++ const bool scalar_quant = ++ x->type == GGML_TYPE_Q8_0 || x->type == GGML_TYPE_Q4_0 || x->type == GGML_TYPE_Q4_1 || ++ x->type == GGML_TYPE_Q5_0 || x->type == GGML_TYPE_Q5_1 || ++ // opencoti-hook: DCA all-KV #444 — q6_0 (added later, #512) lifts via f32 like the other ++ // scalar quants. Omitting it here sent q6_0 through the direct ggml_cast(q6_0->f16) below, ++ // which has no CUDA cpy kernel (only q6_0<->f32 is wired) -> CPU dup_from_q abort at ++ // ggml-cpu/ops.cpp:594. q6_0<->f32 cpy IS GPU-supported, so the two-leg lift stays on GPU. ++ x->type == GGML_TYPE_Q6_0; ++ if (scalar_quant) { ++ x = ggml_cast(ctx0, x, GGML_TYPE_F32); ++ } ++ return ggml_cast(ctx0, x, GGML_TYPE_F16); ++} ++ + ggml_tensor * llm_graph_context::build_attn_dca_core( + const llama_kv_cache_context * mctx_cur, + llm_graph_input_dca * inp_dca, +@@ -190,12 +219,24 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // kept correct under context-shift by the DCA-aware modular K-shift. So read it DIRECTLY — no + // per-step full-cache rope. That re-rope of the entire cache every graph eval was the + // O(n_kv)/token decode bottleneck (gen halving per context-doubling). get_k returns +- // [n_embd_head, n_head_kv, n_kv, 1]. DCA requires an f16 K cache (no q8_0/turbo on this path). +- ggml_tensor * k_dca = mctx_cur->get_k(ctx0, il); +- GGML_ASSERT(k_dca->type == GGML_TYPE_F16 && "dca: requires an f16 K cache (no q8_0/turbo KV)"); ++ // [n_embd_head, n_head_kv, n_kv, 1]. ++ // opencoti F5 dca #444 (DCA all-KV C1): the fused DCA kernel is f16-only by construction, so a ++ // quantized/bf16/turbo K cache is DEQUANTIZED-ON-LIFT to a transient f16 here (the cache itself ++ // stays quantized — that is the memory win; dca_lift_to_f16 only materializes an f16 copy for ++ // the kernel). At f16 this is a no-op (byte-identical to the prior assert path). Context-shift ++ // stays correct independently: the modular K-shift re-ropes the STORED, still-quantized cache ++ // via build_rope_shift's existing dequant->rope->requant quantized branch (llama-kv-cache.cpp). ++ ggml_tensor * k_dca = dca_lift_to_f16(ctx0, mctx_cur->get_k(ctx0, il)); + + // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. ++ // opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16 ++ // while it is still CONTIGUOUS — a permuted view of a block type is not cleanly cpy-able (the ++ // block stride spans dim-0), mirroring build_attn_mha's pre-permute turbo cast. f32 V keeps the ++ // post-permute cast below (unchanged); f16 V is untouched (byte-identical). + ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { ++ v = dca_lift_to_f16(ctx0, v); ++ } + const bool v_trans = v->nb[1] > v->nb[2]; + ggml_tensor * vp = ggml_permute(ctx0, v, 0, 2, 1, 3); + if (v_trans) { +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index 92e3f0b48..2fe40de6f 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2066,7 +2066,11 @@ ggml_tensor * llm_graph_context::build_attn_mha( + // those still cast turbo→f16 up front (ggml_cuda_cpy → ggml_cpy_turbo_f16_cuda, WHT-once + + // InnerQ, VRAM-resident). A single Gemma model thus runs fused SWA layers + materialized global. + const auto is_turbo_vec = [](enum ggml_type t) { +- return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0; ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 ++ // opencoti-hook: TCQ FA-VEC decode (#447/#500). Trellis turbo joins the fused FA-VEC ++ // predicate: K/V are FWHT-rotated (no InnerQ), so the same graph Q-rotation (ggml_turbo_wht ++ // dir=0, scale=nullptr → pure FWHT since InnerQ is inactive) + output inverse-WHT apply. ++ || t == GGML_TYPE_TURBO3_TCQ || t == GGML_TYPE_TURBO2_TCQ; + }; + // opencoti-hook: turboquant perf-lift (M6-S2 Level B, L4.5 AB-prefill-hybrid). The fused FA-VEC + // turbo read wins at DECODE (in-register, bandwidth-bound: turbo2/3/4 = 56/51/48 tps vs 43 +@@ -2088,12 +2092,26 @@ ggml_tensor * llm_graph_context::build_attn_mha( + // scales with n_kv). turbo4/turbo8 at 512 still take the materialize fallback (no D=512 instance). + // The WHT is block-diagonal over 128-blocks, so 512 = 4 Parseval-exact 128-blocks → exact. + const bool fused_turbo_512 = +- k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0); ++ k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0 ++ || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ); // opencoti TCQ #447/#500 (Gemma-4 GLOBAL) ++ // opencoti-hook: TCQ FA-VEC decode (#447/#500). TCQ has NO materialize-lift (turbo-cpy.cu has no ++ // TCQ→f16 inverse-FWHT cast yet), so it must ALWAYS fuse — prefill (large n_q) rides the VEC-tiled ++ // path (correct, slower than MMA; a prefill-MMA dequant is a deferred perf follow-on, like buun's ++ // k_turbo3_tcq_dequant_f16). Without this, a prefill ubatch would fall to the else-branch with no ++ // cast and emit raw TCQ to FA with an UN-rotated Q → wrong logits. ++ const bool is_tcq_kv = (k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ); // opencoti-hook: TCQ FA-VEC (#447/#500) — no materialize-lift, always fuse ++ // opencoti-hook: asymmetric TCQ (#513). TCQ has no f16 materialize-lift, so the only correct path is ++ // the in-register fused FA-VEC read. Allow K!=V when BOTH sides are TCQ (e.g. turbo3_tcq K / turbo2_tcq ++ // V): K and V are FWHT-rotated identically, so the same Q-rotation + output inverse-WHT stay logit-exact ++ // regardless of per-side trellis bit-width. Backed by the dedicated asymmetric FA-VEC instances ++ // (fattn-vec-instance-turbo3_tcq-turbo2_tcq + reverse); the CUDA selector K!=V gate is relaxed to match. ++ const bool both_tcq = (k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) && ++ (v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ); + const bool fused_turbo = + cparams.flash_attn && kq_b == nullptr && +- is_turbo_vec(k->type) && k->type == v->type && ++ is_turbo_vec(k->type) && is_turbo_vec(v->type) && (k->type == v->type || both_tcq) && + (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) && +- tbv_nq_per_strm <= tbv_fuse_max_nq; ++ (tbv_nq_per_strm <= tbv_fuse_max_nq || is_tcq_kv); + + if (fused_turbo) { + // forward-rotate Q (×scale_inv → signs₁ → WHT → 1/√G·signs₂); group auto-picks 128 since +@@ -2856,7 +2874,8 @@ ggml_tensor * llm_graph_context::build_attn_mtp( + ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); + ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); + +- if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) { ++ if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0 ++ || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) + // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the + // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp + // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at +@@ -2875,7 +2894,8 @@ ggml_tensor * llm_graph_context::build_attn_mtp( + ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); + cb(cur, "kqv_out_mtp", il_mtp); + +- if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0) { ++ if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0 ++ || v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) + const int64_t orig_v_head = kv_embd_head_v; + const int64_t padded_v_head = v->ne[0]; + if (padded_v_head != orig_v_head) { +diff --git a/llama.cpp/tools/perplexity/main.cpp b/llama.cpp/tools/perplexity/main.cpp +index 13a9940e9..0f1e79fef 100644 +--- a/llama.cpp/tools/perplexity/main.cpp ++++ b/llama.cpp/tools/perplexity/main.cpp +@@ -1,5 +1,14 @@ ++// opencoti (#517 F-E): trigger llamafile's GPU bootstrap (dlopen CUDA DSO + ggml_backend_register) ++// before the upstream perplexity entry, mirroring tools/server/server.cpp main()'s llamafile_has_gpu() ++// call. The 0.10.3 bump replaced llamafile's patched perplexity main with this thin upstream wrapper, ++// which never registers the llamafile CUDA backend -> the tool runs CPU-only ("no usable GPU found") ++// and CUDA-only turbo/TCQ ops (TURBO_WHT) abort. FLAG_gpu defaults to AUTO, so the detect call alone ++// loads+registers the backend (same as the server). Eval-only tool; see docs §5a-TCQ / buglog. ++extern "C" bool llamafile_has_gpu(void); ++ + int llama_perplexity(int argc, char ** argv); + + int main(int argc, char ** argv) { ++ llamafile_has_gpu(); + return llama_perplexity(argc, argv); + } +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +index 87279b8..4781516 100644 +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -154,7 +154,39 @@ collect_gpu_sources() { + if [ "$fa_all_quants" != "1" ]; then + for f in "$ti_dir"/fattn-vec-instance-turbo2_0-turbo2_0.cu \ + "$ti_dir"/fattn-vec-instance-turbo3_0-turbo3_0.cu \ +- "$ti_dir"/fattn-vec-instance-turbo4_0-turbo4_0.cu; do ++ "$ti_dir"/fattn-vec-instance-turbo4_0-turbo4_0.cu \ ++ "$ti_dir"/fattn-vec-instance-turbo3_tcq-turbo3_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-turbo2_tcq-turbo2_tcq.cu; do ++ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" ++ done ++ fi ++ ++ # 3c. opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). q6_0 is a scalar KV type read by the ++ # FA-VEC path; its 5 Anbeeld instances must compile in the DEFAULT build (same #468/bug-339 ++ # reasoning as the turbo block above — the default build does NOT glob fattn-vec, so without ++ # this the DSO lacks ggml_cuda_flash_attn_ext_vec_case and dlopen fails the instant ++ # q6_0 KV reaches flash-attention). No-op under FA_ALL_QUANTS=1 (already globbed) and on a ++ # vanilla tree (per-file [ -f ] guard). ++ if [ "$fa_all_quants" != "1" ]; then ++ for f in "$ti_dir"/fattn-vec-instance-q6_0-q6_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q8_0-q6_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-q5_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-f16.cu \ ++ "$ti_dir"/fattn-vec-instance-f16-q6_0.cu; do ++ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" ++ done ++ fi ++ ++ # 3d. opencoti-hook: asymmetric TCQ (#513). The mixed turbo3_tcq/turbo2_tcq FA-VEC instances (both ++ # directions) read TCQ K from one tier and TCQ V from the other; like the symmetric TCQ block (3b) ++ # these are FUSED in-register (no dequant-on-lift inverse for TCQ exists), so they MUST compile in ++ # the DEFAULT build — without them the DSO lacks ggml_cuda_flash_attn_ext_vec_case (and reverse) and dlopen fails the instant asymmetric TCQ KV reaches flash-attention ++ # (same #468/bug-339 reasoning). No-op under FA_ALL_QUANTS=1 (already globbed) and on a vanilla ++ # tree (per-file [ -f ] guard). ++ if [ "$fa_all_quants" != "1" ]; then ++ for f in "$ti_dir"/fattn-vec-instance-turbo3_tcq-turbo2_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-turbo2_tcq-turbo3_tcq.cu; do + [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" + done + fi diff --git a/patches/0082-cuda-build-parallelism.patch b/patches/0082-cuda-build-parallelism.patch new file mode 100644 index 0000000000000000000000000000000000000000..88474e732191fd450ba2947bcc0aa7f63763212a --- /dev/null +++ b/patches/0082-cuda-build-parallelism.patch @@ -0,0 +1,320 @@ +diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh +--- a/llamafile/cuda.sh ++++ b/llamafile/cuda.sh +@@ -259,6 +259,22 @@ + COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS" + fi + ++# opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles ++# the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left ++# alone at the build tail uses one core while the rest idle. --threads compiles the arches ++# concurrently. To avoid oversubscribing the bulk (JOBS nvcc already saturate the cores), we divide ++# the outer job-slot count by NVCC_THREADS so PEAK concurrency stays ≈ JOBS (slots × threads). The ++# Windows cuda.bat already passes --threads 5; the Linux path lacked it. Tunable: NVCC_THREADS=1 ++# disables (restores pre-#531 behavior). See docs/protocols/LLAMAFILE_UPGRADE.md. ++NVCC_THREADS="${NVCC_THREADS:-2}" ++JOBS_EFF="$JOBS" ++if [ "${NVCC_THREADS}" -gt 1 ] 2>/dev/null; then ++ COMMON_FLAGS="$COMMON_FLAGS --threads ${NVCC_THREADS}" ++ JOBS_EFF=$(( JOBS / NVCC_THREADS )) ++ [ "$JOBS_EFF" -lt 1 ] && JOBS_EFF=1 ++fi ++echo " nvcc --threads ${NVCC_THREADS} → ${JOBS_EFF} job-slots × ${NVCC_THREADS} arch-threads (#531)" ++ + # Collect sources + collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_QUANTS" + echo " Sources: $NUM_SOURCES .cu files" +@@ -267,7 +283,7 @@ + START_TIME=$(date +%s) + + # Compile GPU sources +-compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS" ++compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" + + COMPILE_TIME=$(date +%s) + echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds" +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -2600,3 +2600,41 @@ + extern DECL_FATTN_MMA_F16_CASE(576, 512, 16, 4); + extern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 32); + extern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32); ++ ++// opencoti-hook: cuda-build-parallelism (#531) — shard the DCA-fused MMA case into parallel ++// instance TUs (template-instances/fattn-mma-f16-dca-fused-instance-*.cu), mirroring the stock ++// DECL_FATTN_MMA_F16_CASE sharding above. Before this, fattn.cu's dca-fused dispatch ++// (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch) instantiated all 48 ++// ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case INLINE — DKQ==DV ∈ {128,256,512} ++// × ncols2 ∈ {1,2,4,8} × ncols1 ∈ {8,16,32,64}/ncols2 → 576 device kernels in ONE TU = a ~52-min ++// SOLO compile tail (fattn.o was 29 MB, the largest object, finishing ~52 min after every other TU). ++// These extern decls move the instantiation to the parallel bulk; fattn.cu now just references the ++// symbols (the regular MMA/TILE/VEC cases were already sharded this way; only our DCA kernel wasn't). ++// The instance set matches the dispatch 1:1 — every reachable _case has a ++// definition, none extra — so the linked DSO is byte-for-byte equivalent (same 576 kernels, relocated). ++#define DECL_FATTN_MMA_F16_DCA_FUSED_CASE(DKQ, DV, ncols1, ncols2) \ ++ template void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case \ ++ (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ ++ ++// DCA always uses DKQ==DV ∈ {128,256,512}; one extern per (ncols1,ncols2) across all three head dims. ++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(ncols1, ncols2) \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, ncols1, ncols2); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, ncols1, ncols2); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, ncols1, ncols2); \ ++ ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(64, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 1, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 8) +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 2); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 4); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 1, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 1, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 1, 8); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 4); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 8); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 2); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 2); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 4); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 8); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 64, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 64, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 64, 1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 1); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 2); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 2); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 4); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 4); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu +@@ -0,0 +1,10 @@ ++// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of ++// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. ++// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. ++// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). ++ ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 8); ++DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 8); diff --git a/patches/0083-tcq-warp-encode.patch b/patches/0083-tcq-warp-encode.patch new file mode 100644 index 0000000000000000000000000000000000000000..25bbf2d3d636d472782b93bda092efb174526355 --- /dev/null +++ b/patches/0083-tcq-warp-encode.patch @@ -0,0 +1,868 @@ +diff --git a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +--- a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +@@ -748,33 +748,17 @@ + const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t); + const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0; + +- // one-time per-device: prefer the 8 KB/block shared-memory backtrace (opt-in); else global fallback buffer +- static int tcq3_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {}; +- static bool tcq3_bt_checked [GGML_CUDA_MAX_DEVICES] = {}; +- constexpr int tcq3_bt_shared_bytes = 128 * 64; +- if (!tcq3_bt_checked[ctx.device]) { +- tcq3_bt_checked[ctx.device] = true; +- const char * env = getenv("TURBO_TCQ_SHARED_BT"); +- if (!env || atoi(env) != 0) { +- int max_shared_optin = 0; +- CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device)); +- if (max_shared_optin >= tcq3_bt_shared_bytes) { +- if (tcq_fp64_enabled()) +- CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo3_tcq), tcq3_bt_shared_bytes); +- else +- CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo3_tcq, tcq3_bt_shared_bytes); +- tcq3_use_shared_bt[ctx.device] = 1; +- } +- } +- } +- if (!tcq3_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); ++ // #533 warp-synchronous encode: bt is always a global per-group slab. With TCQ3_WPB groups/block a shared-bt ++ // would cost WPB*8KB; and the warp kernel is no longer barrier-bound, so global bt is free here. One 8KB slab ++ // per group (indexed by `group`, not blockIdx.x). tcq_bt_buf grows monotonically and is reused across launches. ++ ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); + + const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00); + const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01); + const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02); + const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); + const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); +- const int shared_bytes = tcq3_use_shared_bt[ctx.device] ? tcq3_bt_shared_bytes : 0; ++ const int tcq3_grid = (int)((ne_total_groups + TCQ3_WPB - 1) / TCQ3_WPB); // #533 warp-packed: TCQ3_WPB groups/block, 32 thr/group + // opencoti-hook: TCQ activation-dump (F-D, #515) — bind the staging buffer for this launch (no-op unless + // TURBO_TCQ_DUMP_ACTS is set). tcq_dump_init() is idempotent; dump_dev is nullptr when inactive/cap-reached, + // in which case d_tcq_dump_buf is left at its nullptr default and the kernel's dump branch is a uniform no-op. +@@ -782,15 +766,15 @@ + float * dump_dev = tcq_dump_pre_launch((long) ne_total_groups); + if (dump_dev) CUDA_CHECK(cudaMemcpyToSymbol(d_tcq_dump_buf, &dump_dev, sizeof(float*))); + if (tcq_fp64_enabled()) +- k_set_rows_turbo3_tcq<<<(int)ne_total_groups, 512, shared_bytes, stream>>>( ++ k_set_rows_turbo3_tcq<<>>( + src0_d, src1_d, (block_turbo3_tcq *)dst->data, +- ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13, + s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, + ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); + else +- k_set_rows_turbo3_tcq<<<(int)ne_total_groups, 512, shared_bytes, stream>>>( ++ k_set_rows_turbo3_tcq<<>>( + src0_d, src1_d, (block_turbo3_tcq *)dst->data, +- ne_total_groups, tcq_bt_buf[ctx.device], tcq3_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13, + s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, + ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); + if (dump_dev) tcq_dump_post_launch((long) ne_total_groups, stream); +@@ -827,42 +811,26 @@ + const int64_t s10_i = nb10/sizeof(idx_t), s11_i = nb11/sizeof(idx_t), s12_i = nb12/sizeof(idx_t); + const int iq_is_k = (strncmp(dst->name, "cache_k_", 8) == 0) ? 1 : 0; + +- static int tcq2_use_shared_bt[GGML_CUDA_MAX_DEVICES] = {}; +- static bool tcq2_bt_checked [GGML_CUDA_MAX_DEVICES] = {}; +- constexpr int tcq2_bt_shared_bytes = 128 * 64; +- if (!tcq2_bt_checked[ctx.device]) { +- tcq2_bt_checked[ctx.device] = true; +- const char * env = getenv("TURBO_TCQ_SHARED_BT"); +- if (!env || atoi(env) != 0) { +- int max_shared_optin = 0; +- CUDA_CHECK(cudaDeviceGetAttribute(&max_shared_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, ctx.device)); +- if (max_shared_optin >= tcq2_bt_shared_bytes) { +- if (tcq_fp64_enabled()) +- CUDA_SET_SHARED_MEMORY_LIMIT((k_set_rows_turbo2_tcq), tcq2_bt_shared_bytes); +- else +- CUDA_SET_SHARED_MEMORY_LIMIT(k_set_rows_turbo2_tcq, tcq2_bt_shared_bytes); +- tcq2_use_shared_bt[ctx.device] = 1; +- } +- } +- } +- if (!tcq2_use_shared_bt[ctx.device]) ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); ++ // #533 warp-synchronous encode: bt always a global per-group slab (TCQ2_WPB groups/block); warp kernel is ++ // no longer barrier-bound, so global bt is free. One 8KB slab per group, indexed by `group`. ++ ensure_tcq_bt_buf(ctx.device, ne_total_groups * 128 * 64); + + const uint3 ne00_fd = init_fastdiv_values((uint32_t) ne00); + const uint3 ne01_fd = init_fastdiv_values((uint32_t) ne01); + const uint3 ne02_fd = init_fastdiv_values((uint32_t) ne02); + const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); + const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); +- const int shared_bytes = tcq2_use_shared_bt[ctx.device] ? tcq2_bt_shared_bytes : 0; ++ const int tcq2_grid = (int)((ne_total_groups + TCQ2_WPB - 1) / TCQ2_WPB); // #533 warp-packed: TCQ2_WPB groups/block, 32 thr/group + if (tcq_fp64_enabled()) +- k_set_rows_turbo2_tcq<<<(int)ne_total_groups, 256, shared_bytes, stream>>>( ++ k_set_rows_turbo2_tcq<<>>( + src0_d, src1_d, (block_turbo2_tcq *)dst->data, +- ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13, + s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, + ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); + else +- k_set_rows_turbo2_tcq<<<(int)ne_total_groups, 256, shared_bytes, stream>>>( ++ k_set_rows_turbo2_tcq<<>>( + src0_d, src1_d, (block_turbo2_tcq *)dst->data, +- ne_total_groups, tcq_bt_buf[ctx.device], tcq2_use_shared_bt[ctx.device], ne00, ne01, ne02, ne10, ne11, ne12, ne13, ++ ne_total_groups, tcq_bt_buf[ctx.device], 0, ne00, ne01, ne02, ne10, ne11, ne12, ne13, + s01_f, s02_f, s03_f, s10_i, s11_i, s12_i, iq_is_k, nb1, nb2, nb3, + ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +@@ -109,11 +109,16 @@ + static __device__ __forceinline__ float tcq_acc_sqrt(float v) { return sqrtf(v); } + static __device__ __forceinline__ double tcq_acc_sqrt(double v) { return sqrt(v); } + +-// TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis +-// 512 threads per block (one per trellis state), one block per 128-element group +-// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384) ++// TCQ SET_ROWS encode: warp-synchronous Viterbi (#533). ncu proved the old 512-thread/__syncthreads design was ++// LATENCY-bound on the 128-step block-barrier chain (25% SM, 0.1% DRAM, 33% occ); #538's occupancy/shuffle tweak ++// couldn't help that. This collapses each 128-element group to ONE 32-thread warp (16 states/lane, thread-local ++// predecessor-min, __syncwarp broadcast, bt in global), packs TCQ3_WPB warps/block, and breaks the warp lockstep ++// so per-group serial latency is hidden by other warps. Validated byte-equivalent (round-trip decode cosine ++// 1.00000 vs the old design — .opencoti/tcq-warp-enc-full.cu) + 3.74x on the full encode. Reuses #538's warp- ++// shuffle min primitive. acc_t fp64 path (#506) preserved; bt is always global (use_shared_bt ignored). ++#define TCQ3_WPB 8 + template +-static __global__ void __launch_bounds__(512, 1) k_set_rows_turbo3_tcq( ++static __global__ void __launch_bounds__(TCQ3_WPB*32, 3) k_set_rows_turbo3_tcq( + const float * __restrict__ src0, const idx_t * __restrict__ src1, + block_turbo3_tcq * __restrict__ dst, const int64_t ne_total_groups, + uint8_t * __restrict__ bt_buf, +@@ -127,12 +132,13 @@ + const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd, + const uint3 ne11_fd, const uint3 ne12_fd) { + +- const int64_t group = blockIdx.x; ++ const int warp = threadIdx.x >> 5; ++ const int lane = threadIdx.x & 31; ++ const int64_t group = (int64_t)blockIdx.x * TCQ3_WPB + warp; + if (group >= ne_total_groups) return; ++ (void) use_shared_bt; (void) ne10; (void) ne13; // bt is always global in the warp design + +- const int sid = threadIdx.x; // state index 0..511 +- +- // Compute source and destination pointers (same index math as turbo3) ++ // Compute source and destination pointers (same index math as the 512-thread design; per-warp group) + const int64_t i_base = group * QK_TURBO3_TCQ; + uint32_t tmp = (uint32_t)i_base; uint2 div_mod; + div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x; +@@ -145,244 +151,130 @@ + block_turbo3_tcq * dst_blk = (block_turbo3_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3) + + (i00 / QK_TURBO3_TCQ); + +- // Shared memory layout (~5KB, was ~35KB before global bt optimization): +- // x[128] : rotated+normalized input (also reused as outputs[] after Viterbi) +- // cost[512] : path costs buffer A (also reused for reductions) +- // cost_b[512]: path costs buffer B (double-buffering eliminates 2/3 of syncs) +- // Backtrace: one predecessor byte for each of the 64 low-state groups per +- // step. The predecessor is independent of the output bits in sid[8:6], so +- // storing 128×64 bytes is equivalent to the older 128×512 layout. +- extern __shared__ uint8_t bt_shared[]; +- __shared__ acc_t x[128]; +- __shared__ acc_t cost[512]; +- __shared__ acc_t cost_b[512]; +- __shared__ int warp_min_idx[16]; +- __shared__ acc_t warp_min_cost[16]; +- __shared__ acc_t pred_min_cost[64]; +- __shared__ int shared_initial_state; +- +- if (sid < 128) x[sid] = grp_src[sid]; +- __syncthreads(); +- +- __syncthreads(); +- +- // Norm reduction +- cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f; +- __syncthreads(); +- for (int stride = 256; stride >= 32; stride >>= 1) { +- if (sid < stride) cost[sid] += cost[sid + stride]; +- __syncthreads(); +- } +- if (sid < 32) { +- acc_t v = cost[sid]; +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); +- if (sid == 0) cost[0] = v; +- } +- __syncthreads(); +- acc_t grp_norm = tcq_acc_sqrt(cost[0]); +- acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f; +- +- if (sid < 128) x[sid] *= inv_norm; +- __syncthreads(); +- +- // FWHT. The first five stages are contained within each 32-lane warp, so +- // use warp shuffles and only synchronize for the two cross-warp stages. +- if (sid < 128) { +- acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid]; +- const int lane = sid & 31; +-#pragma unroll +- for (int h = 1; h < 32; h <<= 1) { +- const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h); +- v = (lane & h) ? (other - v) : (v + other); +- } +- x[sid] = v; +- } +- __syncthreads(); +- if (sid < 64) { +- const int j = ((sid >> 5) << 6) + (sid & 31); +- acc_t a = x[j], b = x[j + 32]; +- x[j] = a + b; x[j + 32] = a - b; +- } +- __syncthreads(); +- if (sid < 64) { +- acc_t a = x[sid], b = x[sid + 64]; +- x[sid] = a + b; x[sid + 64] = a - b; ++ // Per-warp smem: whitened vec x[128] (reused as outputs[] after Viterbi) + pred_min_cost[64] broadcast + ++ // initial-state scratch. The Viterbi path-cost lives in registers (c[16], 16 states/lane). bt is global. ++ __shared__ acc_t x_s[TCQ3_WPB][128]; ++ __shared__ acc_t pm_s[TCQ3_WPB][64]; ++ __shared__ int ini_s[TCQ3_WPB]; ++ acc_t * x = x_s[warp]; ++ acc_t * pred_min_cost = pm_s[warp]; ++ uint8_t * bt = bt_buf + group * (int64_t)(128 * 64); // global per-group backtrace slab ++ ++ // --- load 4 elements/lane (stride-32 layout: lane owns positions L, L+32, L+64, L+96) --- ++ acc_t e0 = grp_src[lane], e1 = grp_src[lane+32], e2 = grp_src[lane+64], e3 = grp_src[lane+96]; ++ // --- norm (warp-shuffle sum of squares) --- ++ acc_t ss = e0*e0 + e1*e1 + e2*e2 + e3*e3; ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xFFFFFFFFu, ss, o); ++ acc_t grp_norm = tcq_acc_sqrt(ss); ++ acc_t inv_norm = grp_norm > 1e-10f ? (acc_t)1.0 / grp_norm : (acc_t)0.0; ++ e0 *= inv_norm; e1 *= inv_norm; e2 *= inv_norm; e3 *= inv_norm; ++ // --- FWHT-128 (stride-32 layout: 5 shuffle-butterfly stages + 2 intra-thread cross stages, ++ // byte-faithful to the 512-thread block FWHT) --- ++ e0 *= TURBO3_WHT_SIGNS1[lane]; e1 *= TURBO3_WHT_SIGNS1[lane+32]; ++ e2 *= TURBO3_WHT_SIGNS1[lane+64]; e3 *= TURBO3_WHT_SIGNS1[lane+96]; ++#pragma unroll ++ for (int h = 1; h < 32; h <<= 1) { ++ acc_t o0 = __shfl_xor_sync(0xFFFFFFFFu, e0, h); e0 = (lane & h) ? o0 - e0 : e0 + o0; ++ acc_t o1 = __shfl_xor_sync(0xFFFFFFFFu, e1, h); e1 = (lane & h) ? o1 - e1 : e1 + o1; ++ acc_t o2 = __shfl_xor_sync(0xFFFFFFFFu, e2, h); e2 = (lane & h) ? o2 - e2 : e2 + o2; ++ acc_t o3 = __shfl_xor_sync(0xFFFFFFFFu, e3, h); e3 = (lane & h) ? o3 - e3 : e3 + o3; + } +- __syncthreads(); ++ { acc_t a = e0, b = e1; e0 = a + b; e1 = a - b; acc_t c = e2, d = e3; e2 = c + d; e3 = c - d; } // cross-32: blk0<->1, blk2<->3 ++ { acc_t a = e0, b = e2; e0 = a + b; e2 = a - b; acc_t c = e1, d = e3; e1 = c + d; e3 = c - d; } // cross-64: blk0<->2, blk1<->3 + constexpr float inv_sqrt_128 = 0.08838834764831845f; +- if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid]; +- __syncthreads(); +- +- // opencoti-hook: TCQ activation-dump (F-D, #515). x[0..127] is now the post-FWHT unit-norm whitened vec — +- // exactly the xt the trellis quantizes against d_turbo3_tcq_codebook below, and x stays read-only through the +- // forward pass. Stage it for the F-E codebook retrain when enabled; nullptr (default) is a uniform no-op. +- // group == blockIdx.x < ne_total_groups, so [group*128 + sid] is in-bounds of the (ne_total_groups*128) buffer. +- if (d_tcq_dump_buf != nullptr && sid < 128) { +- d_tcq_dump_buf[group * 128 + sid] = (float) x[sid]; +- } +- +- if (sid == 0) cost[0] = grp_norm; +- __syncthreads(); +- +- acc_t saved_norm = cost[0]; +- +- // Viterbi forward pass: double-buffered cost (1 sync/step, was 3) +- uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64); +- cost[sid] = 0.0f; +- __syncthreads(); +- ++ e0 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane]; e1 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+32]; ++ e2 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+64]; e3 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+96]; ++ x[lane] = e0; x[lane+32] = e1; x[lane+64] = e2; x[lane+96] = e3; ++ __syncwarp(); ++ ++ // opencoti-hook: TCQ activation-dump (F-D, #515). e0..e3 are the post-FWHT unit-norm whitened vec. ++ if (d_tcq_dump_buf != nullptr) { ++ d_tcq_dump_buf[group*128 + lane] = (float) e0; ++ d_tcq_dump_buf[group*128 + lane+32] = (float) e1; ++ d_tcq_dump_buf[group*128 + lane+64] = (float) e2; ++ d_tcq_dump_buf[group*128 + lane+96] = (float) e3; ++ } ++ acc_t saved_norm = grp_norm; ++ ++ // --- Viterbi forward DP: warp-synchronous (16 states/lane, thread-local predecessor-min, __syncwarp). --- ++ // lane L owns states {16L..16L+15} = trellis-groups 2L (states 16L..16L+7) and 2L+1 (states 16L+8..16L+15); ++ // each group's 8 predecessors (g*8..g*8+7) fall entirely within lane g/2, so the predecessor-min is ++ // thread-local (no shuffle). A 64-entry per-warp smem broadcast under __syncwarp replaces the block barrier. ++ acc_t c[16]; ++#pragma unroll ++ for (int i = 0; i < 16; i++) c[i] = 0.0f; + for (int t = 0; t < 128; t++) { +- // Double-buffer: even steps read cost/write cost_b, odd steps read cost_b/write cost +- acc_t * cost_rd = (t & 1) ? cost_b : cost; +- acc_t * cost_wr = (t & 1) ? cost : cost_b; +- ++ acc_t pm0 = c[0]; int pp0 = 0; ++#pragma unroll ++ for (int j = 1; j < 8; j++) { if (c[j] < pm0) { pm0 = c[j]; pp0 = j; } } ++ acc_t pm1 = c[8]; int pp1 = 0; ++#pragma unroll ++ for (int j = 1; j < 8; j++) { if (c[8+j] < pm1) { pm1 = c[8+j]; pp1 = j; } } ++ pred_min_cost[2*lane] = pm0; pred_min_cost[2*lane+1] = pm1; ++ bt[t*64 + 2*lane] = (uint8_t) pp0; bt[t*64 + 2*lane+1] = (uint8_t) pp1; ++ __syncwarp(); + acc_t xt = x[t]; +- +- // Right-shift trellis: ns = (prev >> 3) | (out << 6). The best +- // predecessor depends only on sid's low 6 bits, so compute those 64 +- // minima once instead of repeating the same 8-way scan for each out. +- if (sid < 64) { +- const int base_prev = sid << 3; +- acc_t best = cost_rd[base_prev]; +- int best_p = 0; +-#pragma unroll +- for (int p = 1; p < 8; p++) { +- acc_t c = cost_rd[base_prev | p]; +- if (c < best) { +- best = c; +- best_p = p; +- } +- } +- pred_min_cost[sid] = best; +- bt[t * 64 + sid] = (uint8_t) best_p; +- } +- __syncthreads(); +- +- const int pred_idx = sid & 0x3F; +- acc_t dist = xt - d_turbo3_tcq_codebook[sid]; +- dist = dist * dist; +- +- cost_wr[sid] = pred_min_cost[pred_idx] + dist; +- __syncthreads(); +- } +- // After 128 steps (even count): final costs are in cost[] (step 127 writes to cost) +- +- // Warp argmin over 512 costs +- { +- acc_t my_cost = cost[sid]; +- int my_idx = sid; +- #pragma unroll +- for (int offset = 16; offset > 0; offset >>= 1) { +- acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset); +- int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset); +- if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; } +- } +- if (sid % 32 == 0) { +- warp_min_cost[sid / 32] = my_cost; +- warp_min_idx[sid / 32] = my_idx; +- } ++#pragma unroll ++ for (int i = 0; i < 16; i++) { int s = 16*lane + i; acc_t d = xt - d_turbo3_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; } ++ __syncwarp(); + } +- __syncthreads(); +- if (sid < 32) { +- acc_t best = (sid < 16) ? warp_min_cost[sid] : 3.4028234663852886e38f; +- int best_idx = (sid < 16) ? warp_min_idx[sid] : 0; +-#pragma unroll +- for (int offset = 16; offset > 0; offset >>= 1) { +- acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset); +- int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset); +- if (other_cost < best) { +- best = other_cost; +- best_idx = other_idx; +- } +- } +- if (sid == 0) { +- shared_initial_state = best_idx; // temporarily: best final state (becomes initial after backtrack) +- } ++ // --- argmin over 512 states (per-lane min over 16, then warp-shuffle) --- ++ acc_t mc = c[0]; int mi = 16*lane; ++#pragma unroll ++ for (int i = 1; i < 16; i++) { if (c[i] < mc) { mc = c[i]; mi = 16*lane + i; } } ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) { ++ acc_t oc = __shfl_xor_sync(0xFFFFFFFFu, mc, o); int oi = __shfl_xor_sync(0xFFFFFFFFu, mi, o); ++ if (oc < mc || (oc == mc && oi < mi)) { mc = oc; mi = oi; } + } +- __syncthreads(); +- +- // Save x[] to global buffer before backtrack overwrites it +- +- // Backtrack (inherently sequential, reads global bt) ++ // --- backtrack (lane 0; outputs reuse the x slab — x no longer needed) --- + uint8_t * outputs = (uint8_t *)x; +- if (sid == 0) { +- int state = shared_initial_state; ++ if (lane == 0) { ++ int state = mi; + for (int t = 127; t >= 0; t--) { + outputs[t] = (uint8_t)(state >> 6); +- int p = bt[t * 64 + (state & 0x3F)]; ++ int p = bt[t*64 + (state & 0x3F)]; + state = ((state & 0x3F) << 3) | p; + } +- shared_initial_state = state; ++ ini_s[warp] = state; + } +- __syncthreads(); ++ __syncwarp(); ++ int initial_state = ini_s[warp]; + +- // Save output symbols to global buffer +- +- // Parallel recon norm: t>=2 can compute state directly from 3 outputs (3 shifts of 3 = 9 bits) +- acc_t my_recon_sq = 0.0f; +- if (sid < 128) { +- int cur_state; +- if (sid < 2) { +- cur_state = shared_initial_state; +- for (int t = 0; t <= sid; t++) +- cur_state = (cur_state >> 3) | (((int)outputs[t]) << 6); +- } else { +- cur_state = ((int)outputs[sid - 2] & 0x7) +- | (((int)outputs[sid - 1] & 0x7) << 3) +- | (((int)outputs[sid] & 0x7) << 6); +- } +- acc_t c = d_turbo3_tcq_codebook[cur_state]; +- my_recon_sq = c * c; +- } +- cost[sid] = my_recon_sq; +- __syncthreads(); +- for (int stride = 256; stride >= 32; stride >>= 1) { +- if (sid < stride) cost[sid] += cost[sid + stride]; +- __syncthreads(); +- } +- if (sid < 32) { +- acc_t v = cost[sid]; +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); +- if (sid == 0) cost[0] = v; ++ // --- recon norm (warp-shuffle sum; lane owns recon positions L, L+32, L+64, L+96) --- ++ acc_t recon_sq = 0.0f; ++#pragma unroll ++ for (int b = 0; b < 4; b++) { ++ int pos = lane + 32*b; int cur_state; ++ if (pos < 2) { cur_state = initial_state; for (int t = 0; t <= pos; t++) cur_state = (cur_state >> 3) | (((int)outputs[t]) << 6); } ++ else { cur_state = ((int)outputs[pos-2] & 0x7) | (((int)outputs[pos-1] & 0x7) << 3) | (((int)outputs[pos] & 0x7) << 6); } ++ acc_t cc = d_turbo3_tcq_codebook[cur_state]; recon_sq += cc * cc; + } +- __syncthreads(); +- acc_t recon_norm = tcq_acc_sqrt(cost[0]); ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) recon_sq += __shfl_xor_sync(0xFFFFFFFFu, recon_sq, o); ++ acc_t recon_norm = tcq_acc_sqrt(recon_sq); + acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm; + corrected_norm *= innerq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v; + +- // Parallel bitpack: qs stores 6 initial-state bits followed by 128 3-bit +- // output symbols. Each byte is independent, so avoid the old serial OR loop. +- if (sid < 49) { +- const int init_bits = (shared_initial_state >> 3) & 0x3F; +- uint8_t packed = 0; +-#pragma unroll +- for (int bit = 0; bit < 8; bit++) { +- const int pos = sid * 8 + bit; +- int v = 0; +- if (pos < 6) { +- v = (init_bits >> pos) & 1; +- } else { +- const int sym_bit_pos = pos - 6; +- const int sym_idx = sym_bit_pos / 3; +- if (sym_idx < 128) { +- v = (outputs[sym_idx] >> (sym_bit_pos % 3)) & 1; +- } ++ // --- bitpack: lane handles bytes {lane, lane+32} that are < 49 --- ++ const int init_bits = (initial_state >> 3) & 0x3F; ++#pragma unroll ++ for (int bb = 0; bb < 2; bb++) { ++ int byte = lane + 32*bb; ++ if (byte < 49) { ++ uint8_t packed = 0; ++#pragma unroll ++ for (int bit = 0; bit < 8; bit++) { ++ const int pos = byte*8 + bit; int v = 0; ++ if (pos < 6) v = (init_bits >> pos) & 1; ++ else { const int sym_bit_pos = pos - 6; const int sym_idx = sym_bit_pos / 3; if (sym_idx < 128) v = (outputs[sym_idx] >> (sym_bit_pos % 3)) & 1; } ++ packed |= (uint8_t)(v << bit); + } +- packed |= (uint8_t)(v << bit); ++ dst_blk->qs[byte] = packed; + } +- dst_blk->qs[sid] = packed; +- } +- if (sid == 0) { +- dst_blk->norm = __float2half((float)corrected_norm); + } ++ if (lane == 0) dst_blk->norm = __float2half((float)corrected_norm); + } + + // TCQ GET_ROWS dequantize (for non-FA paths) +@@ -455,10 +347,13 @@ + -0.13432936f, -0.05269006f, +0.03536416f, +0.117640756f, -0.022776067f, +0.042032316f, +0.10472976f, +0.18042557f + }; + +-// 2-bit TCQ SET_ROWS encode: Viterbi optimal path with right-shift trellis (k=2, L=8) +-// Double-buffered cost arrays + global memory backtrace (128 syncs/group, was 384) ++// 2-bit TCQ SET_ROWS encode: warp-synchronous Viterbi (#533, k=2, L=8, 256 states). Same warp redesign as ++// turbo3_tcq: 1 group / 32-thread warp (8 states/lane, thread-local quartet predecessor-min, __syncwarp ++// broadcast, bt in global), TCQ2_WPB warps/block. Replaces the old 256-thread/__syncthreads block (latency- ++// bound on the 128-step barrier chain). bt always global (use_shared_bt ignored); acc_t fp64 path preserved. ++#define TCQ2_WPB 8 + template +-static __global__ void __launch_bounds__(256, 1) k_set_rows_turbo2_tcq( ++static __global__ void __launch_bounds__(TCQ2_WPB*32, 3) k_set_rows_turbo2_tcq( + const float * __restrict__ src0, const idx_t * __restrict__ src1, + block_turbo2_tcq * __restrict__ dst, const int64_t ne_total_groups, + uint8_t * __restrict__ bt_buf, +@@ -472,12 +367,14 @@ + const uint3 ne00_fd, const uint3 ne01_fd, const uint3 ne02_fd, + const uint3 ne11_fd, const uint3 ne12_fd) { + +- const int grp = blockIdx.x; +- if (grp >= ne_total_groups) return; +- const int sid = threadIdx.x; // 0..255 = trellis state ++ const int warp = threadIdx.x >> 5; ++ const int lane = threadIdx.x & 31; ++ const int64_t group = (int64_t)blockIdx.x * TCQ2_WPB + warp; ++ if (group >= ne_total_groups) return; ++ (void) use_shared_bt; (void) ne10; (void) ne13; // bt is always global in the warp design + +- // Compute source and destination pointers (all threads, used by thread 0) +- const int64_t i_base = int64_t(grp) * QK_TURBO2_TCQ; ++ // Compute source and destination pointers (same index math as the 256-thread design; per-warp group) ++ const int64_t i_base = group * QK_TURBO2_TCQ; + uint32_t tmp = (uint32_t)i_base; uint2 div_mod; + div_mod = fast_div_modulo(tmp, ne00_fd); const int64_t i00 = div_mod.y; tmp = div_mod.x; + div_mod = fast_div_modulo(tmp, ne01_fd); const int64_t i01 = div_mod.y; tmp = div_mod.x; +@@ -489,232 +386,120 @@ + block_turbo2_tcq * dst_blk = (block_turbo2_tcq *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3) + + (i00 / QK_TURBO2_TCQ); + +- // Backtrace: one predecessor byte per 64 low-state groups per step. +- // The predecessor depends only on sid's low 6 bits (same as turbo3_tcq). +- extern __shared__ uint8_t bt_shared[]; +- __shared__ acc_t x[128]; +- __shared__ acc_t cost[256]; +- __shared__ acc_t cost_b[256]; // double-buffering for Viterbi +- __shared__ int warp_min_idx[8]; +- __shared__ acc_t warp_min_cost[8]; +- __shared__ acc_t pred_min_cost[64]; +- __shared__ int shared_initial_state; +- +- if (sid < 128) x[sid] = grp_src[sid]; +- __syncthreads(); +- +- __syncthreads(); +- +- // Norm reduction +- cost[sid] = (sid < 128) ? x[sid] * x[sid] : 0.0f; +- __syncthreads(); +- for (int stride = 128; stride >= 32; stride >>= 1) { +- if (sid < stride) cost[sid] += cost[sid + stride]; +- __syncthreads(); +- } +- if (sid < 32) { +- acc_t v = cost[sid]; +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); +- if (sid == 0) cost[0] = v; +- } +- __syncthreads(); +- acc_t grp_norm = tcq_acc_sqrt(cost[0]); +- acc_t inv_norm = grp_norm > 1e-10f ? 1.0f / grp_norm : 0.0f; +- +- if (sid < 128) x[sid] *= inv_norm; +- __syncthreads(); +- +- // FWHT. The first five stages use warp shuffles, the two cross-warp stages +- // use shared memory (same approach as turbo3_tcq). +- if (sid < 128) { +- acc_t v = x[sid] * TURBO3_WHT_SIGNS1[sid]; +- const int lane = sid & 31; +-#pragma unroll +- for (int h = 1; h < 32; h <<= 1) { +- const acc_t other = __shfl_xor_sync(0xFFFFFFFFULL, v, h); +- v = (lane & h) ? (other - v) : (v + other); +- } +- x[sid] = v; +- } +- __syncthreads(); +- if (sid < 64) { +- const int j = ((sid >> 5) << 6) + (sid & 31); +- acc_t a = x[j], b = x[j + 32]; +- x[j] = a + b; x[j + 32] = a - b; +- } +- __syncthreads(); +- if (sid < 64) { +- acc_t a = x[sid], b = x[sid + 64]; +- x[sid] = a + b; x[sid + 64] = a - b; ++ // Per-warp smem: whitened vec x[128] (reused as outputs[] after Viterbi) + pred_min_cost[64] broadcast + ++ // initial-state scratch. Path-cost in registers (c[8], 8 states/lane). bt is global. ++ __shared__ acc_t x_s[TCQ2_WPB][128]; ++ __shared__ acc_t pm_s[TCQ2_WPB][64]; ++ __shared__ int ini_s[TCQ2_WPB]; ++ acc_t * x = x_s[warp]; ++ acc_t * pred_min_cost = pm_s[warp]; ++ uint8_t * bt = bt_buf + group * (int64_t)(128 * 64); // global per-group backtrace slab ++ ++ // --- load 4 elements/lane (stride-32 layout: lane owns positions L, L+32, L+64, L+96) --- ++ acc_t e0 = grp_src[lane], e1 = grp_src[lane+32], e2 = grp_src[lane+64], e3 = grp_src[lane+96]; ++ // --- norm (warp-shuffle sum of squares) --- ++ acc_t ss = e0*e0 + e1*e1 + e2*e2 + e3*e3; ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xFFFFFFFFu, ss, o); ++ acc_t grp_norm = tcq_acc_sqrt(ss); ++ acc_t inv_norm = grp_norm > 1e-10f ? (acc_t)1.0 / grp_norm : (acc_t)0.0; ++ e0 *= inv_norm; e1 *= inv_norm; e2 *= inv_norm; e3 *= inv_norm; ++ // --- FWHT-128 (stride-32 layout: 5 shuffle-butterfly stages + 2 intra-thread cross stages) --- ++ e0 *= TURBO3_WHT_SIGNS1[lane]; e1 *= TURBO3_WHT_SIGNS1[lane+32]; ++ e2 *= TURBO3_WHT_SIGNS1[lane+64]; e3 *= TURBO3_WHT_SIGNS1[lane+96]; ++#pragma unroll ++ for (int h = 1; h < 32; h <<= 1) { ++ acc_t o0 = __shfl_xor_sync(0xFFFFFFFFu, e0, h); e0 = (lane & h) ? o0 - e0 : e0 + o0; ++ acc_t o1 = __shfl_xor_sync(0xFFFFFFFFu, e1, h); e1 = (lane & h) ? o1 - e1 : e1 + o1; ++ acc_t o2 = __shfl_xor_sync(0xFFFFFFFFu, e2, h); e2 = (lane & h) ? o2 - e2 : e2 + o2; ++ acc_t o3 = __shfl_xor_sync(0xFFFFFFFFu, e3, h); e3 = (lane & h) ? o3 - e3 : e3 + o3; + } +- __syncthreads(); ++ { acc_t a = e0, b = e1; e0 = a + b; e1 = a - b; acc_t c = e2, d = e3; e2 = c + d; e3 = c - d; } // cross-32 ++ { acc_t a = e0, b = e2; e0 = a + b; e2 = a - b; acc_t c = e1, d = e3; e1 = c + d; e3 = c - d; } // cross-64 + constexpr float inv_sqrt_128 = 0.08838834764831845f; +- if (sid < 128) x[sid] *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[sid]; +- __syncthreads(); +- +- if (sid == 0) cost[0] = grp_norm; +- __syncthreads(); +- +- acc_t saved_norm = cost[0]; +- +- // Viterbi forward pass: double-buffered cost (1 sync/step, was 3) +- uint8_t * bt = use_shared_bt ? bt_shared : bt_buf + (int64_t)blockIdx.x * (128 * 64); +- cost[sid] = 0.0f; +- __syncthreads(); +- ++ e0 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane]; e1 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+32]; ++ e2 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+64]; e3 *= inv_sqrt_128 * TURBO3_WHT_SIGNS2[lane+96]; ++ x[lane] = e0; x[lane+32] = e1; x[lane+64] = e2; x[lane+96] = e3; ++ __syncwarp(); ++ acc_t saved_norm = grp_norm; ++ ++ // --- Viterbi forward DP: warp-synchronous (8 states/lane, thread-local quartet pred-min, __syncwarp). --- ++ // lane L owns states {8L..8L+7} = trellis-groups 2L (states 8L..8L+3) and 2L+1 (states 8L+4..8L+7); each ++ // group's 4 predecessors (g*4..g*4+3) fall within lane g/2, so predecessor-min is thread-local (no shuffle). ++ acc_t c[8]; ++#pragma unroll ++ for (int i = 0; i < 8; i++) c[i] = 0.0f; + for (int t = 0; t < 128; t++) { +- acc_t * cost_rd = (t & 1) ? cost_b : cost; +- acc_t * cost_wr = (t & 1) ? cost : cost_b; +- ++ acc_t pm0 = c[0]; int pp0 = 0; ++#pragma unroll ++ for (int j = 1; j < 4; j++) { if (c[j] < pm0) { pm0 = c[j]; pp0 = j; } } ++ acc_t pm1 = c[4]; int pp1 = 0; ++#pragma unroll ++ for (int j = 1; j < 4; j++) { if (c[4+j] < pm1) { pm1 = c[4+j]; pp1 = j; } } ++ pred_min_cost[2*lane] = pm0; pred_min_cost[2*lane+1] = pm1; ++ bt[t*64 + 2*lane] = (uint8_t) pp0; bt[t*64 + 2*lane+1] = (uint8_t) pp1; ++ __syncwarp(); + acc_t xt = x[t]; +- +- // Right-shift trellis (k=2, L=8): ns = (prev >> 2) | (out << 6). +- // The best predecessor depends only on sid's low 6 bits, so compute +- // those 64 minima once instead of repeating the same 4-way scan per state. +- if (sid < 64) { +- const int base_prev = sid << 2; +- acc_t best = cost_rd[base_prev]; +- int best_p = 0; +-#pragma unroll +- for (int p = 1; p < 4; p++) { +- acc_t c = cost_rd[base_prev | p]; +- if (c < best) { +- best = c; +- best_p = p; +- } +- } +- pred_min_cost[sid] = best; +- bt[t * 64 + sid] = (uint8_t) best_p; +- } +- __syncthreads(); +- +- const int pred_idx = sid & 0x3F; +- acc_t dist = xt - d_turbo2_tcq_codebook[sid]; +- dist = dist * dist; +- +- cost_wr[sid] = pred_min_cost[pred_idx] + dist; +- __syncthreads(); +- } +- // After 128 steps (even count): final costs in cost[] +- +- // Warp argmin over 256 costs +- { +- acc_t my_cost = cost[sid]; +- int my_idx = sid; +- #pragma unroll +- for (int offset = 16; offset > 0; offset >>= 1) { +- acc_t other_cost = __shfl_xor_sync(0xFFFFFFFFULL, my_cost, offset); +- int other_idx = __shfl_xor_sync(0xFFFFFFFFULL, my_idx, offset); +- if (other_cost < my_cost) { my_cost = other_cost; my_idx = other_idx; } +- } +- if (sid % 32 == 0) { +- warp_min_cost[sid / 32] = my_cost; +- warp_min_idx[sid / 32] = my_idx; +- } ++#pragma unroll ++ for (int i = 0; i < 8; i++) { int s = 8*lane + i; acc_t d = xt - d_turbo2_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; } ++ __syncwarp(); + } +- __syncthreads(); +- if (sid < 32) { +- acc_t best = (sid < 8) ? warp_min_cost[sid] : 3.4028234663852886e38f; +- int best_idx = (sid < 8) ? warp_min_idx[sid] : 0; +-#pragma unroll +- for (int offset = 16; offset > 0; offset >>= 1) { +- acc_t other_cost = __shfl_down_sync(0xFFFFFFFFULL, best, offset); +- int other_idx = __shfl_down_sync(0xFFFFFFFFULL, best_idx, offset); +- if (other_cost < best) { +- best = other_cost; +- best_idx = other_idx; +- } +- } +- if (sid == 0) { +- shared_initial_state = best_idx; +- } ++ // --- argmin over 256 states (per-lane min over 8, then warp-shuffle) --- ++ acc_t mc = c[0]; int mi = 8*lane; ++#pragma unroll ++ for (int i = 1; i < 8; i++) { if (c[i] < mc) { mc = c[i]; mi = 8*lane + i; } } ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) { ++ acc_t oc = __shfl_xor_sync(0xFFFFFFFFu, mc, o); int oi = __shfl_xor_sync(0xFFFFFFFFu, mi, o); ++ if (oc < mc || (oc == mc && oi < mi)) { mc = oc; mi = oi; } + } +- __syncthreads(); +- +- // Save x[] to global buffer before backtrack overwrites it +- +- // Backtrack (inherently sequential, reads compressed bt) ++ // --- backtrack (lane 0; outputs reuse the x slab) — turbo2: 2-bit output (state>>6), <<2 trellis shift --- + uint8_t * outputs = (uint8_t *)x; +- if (sid == 0) { +- int state = shared_initial_state; ++ if (lane == 0) { ++ int state = mi; + for (int t = 127; t >= 0; t--) { + outputs[t] = (uint8_t)(state >> 6); +- int p = bt[t * 64 + (state & 0x3F)]; ++ int p = bt[t*64 + (state & 0x3F)]; + state = ((state & 0x3F) << 2) | p; + } +- shared_initial_state = state; ++ ini_s[warp] = state; + } +- __syncthreads(); +- +- // Save output symbols to global buffer ++ __syncwarp(); ++ int initial_state = ini_s[warp]; + +- // Parallel recon norm: t>=3 can compute state directly from 4 outputs (4 shifts of 2 = 8 bits) +- acc_t my_recon_sq = 0.0f; +- if (sid < 128) { +- int cur_state; +- if (sid < 3) { +- cur_state = shared_initial_state; +- for (int t = 0; t <= sid; t++) +- cur_state = (cur_state >> 2) | (((int)outputs[t]) << 6); +- } else { +- cur_state = ((int)outputs[sid - 3] & 0x3) +- | (((int)outputs[sid - 2] & 0x3) << 2) +- | (((int)outputs[sid - 1] & 0x3) << 4) +- | (((int)outputs[sid] & 0x3) << 6); +- } +- acc_t c = d_turbo2_tcq_codebook[cur_state]; +- my_recon_sq = c * c; +- } +- cost[sid] = my_recon_sq; +- __syncthreads(); +- for (int stride = 128; stride >= 32; stride >>= 1) { +- if (sid < stride) cost[sid] += cost[sid + stride]; +- __syncthreads(); +- } +- if (sid < 32) { +- acc_t v = cost[sid]; +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 16); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 8); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 4); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 2); +- v += __shfl_down_sync(0xFFFFFFFFULL, v, 1); +- if (sid == 0) cost[0] = v; ++ // --- recon norm (warp-shuffle sum; turbo2: pos<3 walk, else 4 symbols x 2-bit) --- ++ acc_t recon_sq = 0.0f; ++#pragma unroll ++ for (int b = 0; b < 4; b++) { ++ int pos = lane + 32*b; int cur_state; ++ if (pos < 3) { cur_state = initial_state; for (int t = 0; t <= pos; t++) cur_state = (cur_state >> 2) | (((int)outputs[t]) << 6); } ++ else { cur_state = ((int)outputs[pos-3] & 0x3) | (((int)outputs[pos-2] & 0x3) << 2) | (((int)outputs[pos-1] & 0x3) << 4) | (((int)outputs[pos] & 0x3) << 6); } ++ acc_t cc = d_turbo2_tcq_codebook[cur_state]; recon_sq += cc * cc; + } +- __syncthreads(); +- acc_t recon_norm = tcq_acc_sqrt(cost[0]); ++#pragma unroll ++ for (int o = 16; o > 0; o >>= 1) recon_sq += __shfl_xor_sync(0xFFFFFFFFu, recon_sq, o); ++ acc_t recon_norm = tcq_acc_sqrt(recon_sq); + acc_t corrected_norm = (recon_norm > 1e-10f) ? saved_norm / recon_norm : saved_norm; + corrected_norm *= iq_is_k ? d_tcq_norm_alpha : d_tcq_norm_alpha_v; + +- // Parallel bitpack: qs stores 6 initial-state bits followed by 128 2-bit +- // output symbols. Each byte is independent (2-bit symbols never cross byte +- // boundaries after the 6-bit prefix), so avoid the serial OR loop. +- if (sid < 33) { +- const int init_bits = (shared_initial_state >> 2) & 0x3F; +- uint8_t packed = 0; +-#pragma unroll +- for (int bit = 0; bit < 8; bit++) { +- const int pos = sid * 8 + bit; +- int v = 0; +- if (pos < 6) { +- v = (init_bits >> pos) & 1; +- } else { +- const int sym_bit_pos = pos - 6; +- const int sym_idx = sym_bit_pos / 2; +- if (sym_idx < 128) { +- v = (outputs[sym_idx] >> (sym_bit_pos % 2)) & 1; +- } ++ // --- bitpack: 6 init bits + 128 x 2-bit symbols = 33 bytes; lane handles bytes {lane, lane+32} < 33 --- ++ const int init_bits = (initial_state >> 2) & 0x3F; ++#pragma unroll ++ for (int bb = 0; bb < 2; bb++) { ++ int byte = lane + 32*bb; ++ if (byte < 33) { ++ uint8_t packed = 0; ++#pragma unroll ++ for (int bit = 0; bit < 8; bit++) { ++ const int pos = byte*8 + bit; int v = 0; ++ if (pos < 6) v = (init_bits >> pos) & 1; ++ else { const int sym_bit_pos = pos - 6; const int sym_idx = sym_bit_pos / 2; if (sym_idx < 128) v = (outputs[sym_idx] >> (sym_bit_pos % 2)) & 1; } ++ packed |= (uint8_t)(v << bit); + } +- packed |= (uint8_t)(v << bit); ++ dst_blk->qs[byte] = packed; + } +- dst_blk->qs[sid] = packed; +- } +- if (sid == 0) { +- dst_blk->norm = __float2half((float)corrected_norm); + } ++ if (lane == 0) dst_blk->norm = __float2half((float)corrected_norm); + } + + // 2-bit TCQ GET_ROWS dequantize diff --git a/patches/0084-cuda-graphs.patch b/patches/0084-cuda-graphs.patch new file mode 100644 index 0000000000000000000000000000000000000000..ea42a6afa7bb178bd1a5c33fda4e03a4a3cd1d3b --- /dev/null +++ b/patches/0084-cuda-graphs.patch @@ -0,0 +1,23 @@ +diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh +--- a/llamafile/cuda.sh ++++ b/llamafile/cuda.sh +@@ -259,6 +259,19 @@ + COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS" + fi + ++# opencoti-hook: cuda-graphs (#533 decode-tps lever L6) — enable CUDA graph capture+replay. ++# Upstream llama.cpp's CMake defines GGML_CUDA_USE_GRAPHS by default; the llamafile cuda.sh build ++# omitted it, so the entire `#ifdef USE_CUDA_GRAPH` capture/replay path (ggml-cuda.cu:3318/4478/ ++# 4515/4593/4698; common.cuh:1218) was compiled OUT — every decode step launched hundreds of ++# kernels individually with full per-launch CPU overhead on a latency-bound (~4.5% SM) decode. ++# This flag turns it on; the runtime guards (split buffers, unsupported node types, the opencoti ++# NEO head-split force-disable) fall back to eager capture-by-capture when a graph isn't viable, so ++# it composes with the opencoti stack. Tunable: OPENCOTI_NO_CUDA_GRAPHS=1 omits it (pre-#533). ++# See docs/evaluations/decode-levers.md (L6). ++if [ "${OPENCOTI_NO_CUDA_GRAPHS:-0}" != "1" ]; then ++ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_USE_GRAPHS" ++fi ++ + # opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles + # the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left + # alone at the build tail uses one core while the rest idle. --threads compiles the arches diff --git a/patches/0085-dca-quant-prefill-inlaunch.patch b/patches/0085-dca-quant-prefill-inlaunch.patch new file mode 100644 index 0000000000000000000000000000000000000000..90f1c47d250eece626dce798645311d9289975b9 --- /dev/null +++ b/patches/0085-dca-quant-prefill-inlaunch.patch @@ -0,0 +1,237 @@ +From: opencoti +Subject: [PATCH 0085] DCA quant-KV prefill — in-launch single-pass dequant (#554) + +Move scalar/turbo K/V dequant for the multi-chunk DCA fused flash-attn op out of the +two-stage whole-cache graph lift (dca_lift_to_f16: q->f32->f16 materialized every +forward, GPU-memory-bound and O(cache) — catastrophic at >=512k) and into a single-pass +to_fp16_nc pool convert inside the fused launch (mirrors the non-DCA launch_fattn fast +path). Byte-identical to the prior lift (both compute d*q in f32 then round to f16); +restores DCA + quantized-KV as a first-class long-context path (~12x at 85k, ~30x at +512k: 46 -> 1379+ tok/s, the 1M q5_0/q4_0 path). Also moves the regime==3 fused +early-out above the f16-K asserts in ggml_cuda_flash_attn_ext_lse so quant-K cells reach +the fused path instead of asserting. + +dca_is_inkernel_liftable() gates the defer to Q8_0/Q4_0/Q4_1/Q5_0/Q5_1/Q6_0 (+ the +turbo/TCQ families via the existing to_fp16_nc table); f16/bf16 keep the stock path. +See .wolf/buglog.json bug-720. + +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -204,6 +204,20 @@ + return ggml_cast(ctx0, x, GGML_TYPE_F16); + } + ++// opencoti F5 dca (DCA all-KV C-perf, bug-720): can the FUSED multi-chunk kernel dequant this cache type ++// to f16 INSIDE its launch (single-pass to_fp16_nc into a transient pool buffer, mirroring launch_fattn), ++// instead of dca_lift_to_f16 whole-cache materializing it to f16 in the GRAPH every forward? That graph ++// lift is the prefill cliff: at 98k/q5_0 the two-stage q->f32->f16 cast ran every ubatch and dragged ++// DCA-on prefill to ~138 tps vs ~1212 for the same scalar-quant cache DCA-off (which already converts ++// per-op via launch_fattn). Restrict to the SCALAR quants with a to_fp16_nc converter (convert.cu:843); ++// turbo/TCQ/bf16 keep dca_lift_to_f16 (already a single-pass DIRECT ->f16 cast, not the slow two-stage), ++// f16 is a no-op. The in-op convert is byte-IDENTICAL to the lift (both compute d*q in f32 then round to ++// f16), so this is a pure perf change, gateable by logit-equivalence. ++static bool dca_is_inkernel_liftable(enum ggml_type t) { ++ return t == GGML_TYPE_Q8_0 || t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 || ++ t == GGML_TYPE_Q5_0 || t == GGML_TYPE_Q5_1 || t == GGML_TYPE_Q6_0; ++} ++ + ggml_tensor * llm_graph_context::build_attn_dca_core( + const llama_kv_cache_context * mctx_cur, + llm_graph_input_dca * inp_dca, +@@ -226,24 +240,53 @@ + // the kernel). At f16 this is a no-op (byte-identical to the prior assert path). Context-shift + // stays correct independently: the modular K-shift re-ropes the STORED, still-quantized cache + // via build_rope_shift's existing dequant->rope->requant quantized branch (llama-kv-cache.cpp). +- ggml_tensor * k_dca = dca_lift_to_f16(ctx0, mctx_cur->get_k(ctx0, il)); ++ ggml_tensor * k_raw = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * v_raw = mctx_cur->get_v(ctx0, il); ++ ++ // opencoti F5 dca (DCA all-KV C-perf, bug-720): on the analytical-band (multi-chunk) path the fused ++ // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see ++ // ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case), so pass the quantized cache THROUGH instead of ++ // whole-cache materializing it to f16 via dca_lift_to_f16 every forward (the prefill cliff). Gate: ++ // - !single_chunk: only the fused op carries the in-launch convert; single_chunk uses stock ++ // ggml_flash_attn_ext (launch_fattn already converts scalar-quant natively) and runs only while ++ // n_kv <= chunk (a tiny cache) so its lift cost is negligible — keep it on the safe lift path. ++ // - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read. ++ // - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the ++ // permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift. ++ // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output. ++ const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2]; ++ const bool defer_quant_lift = ++ !inp_dca->single_chunk && ++ dca_is_inkernel_liftable(k_raw->type) && ++ dca_is_inkernel_liftable(v_raw->type) && ++ !v_trans_raw; + + // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. + // opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16 + // while it is still CONTIGUOUS — a permuted view of a block type is not cleanly cpy-able (the + // block stride spans dim-0), mirroring build_attn_mha's pre-permute turbo cast. f32 V keeps the + // post-permute cast below (unchanged); f16 V is untouched (byte-identical). +- ggml_tensor * v = mctx_cur->get_v(ctx0, il); +- if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { +- v = dca_lift_to_f16(ctx0, v); +- } +- const bool v_trans = v->nb[1] > v->nb[2]; +- ggml_tensor * vp = ggml_permute(ctx0, v, 0, 2, 1, 3); +- if (v_trans) { +- vp = ggml_transpose(ctx0, vp); +- } +- if (vp->type == GGML_TYPE_F32) { +- vp = ggml_cast(ctx0, vp, GGML_TYPE_F16); ++ ggml_tensor * k_dca; ++ ggml_tensor * vp; ++ if (defer_quant_lift) { ++ // pass scalar-quant K/V straight to the fused op; permute(0,2,1,3) is a clean view (dim-0 = the ++ // block axis, untouched) that the launch's to_fp16_nc dequants with the permuted strides. ++ k_dca = k_raw; ++ vp = ggml_permute(ctx0, v_raw, 0, 2, 1, 3); ++ } else { ++ k_dca = dca_lift_to_f16(ctx0, k_raw); ++ ggml_tensor * v = v_raw; ++ if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { ++ v = dca_lift_to_f16(ctx0, v); ++ } ++ const bool v_trans = v->nb[1] > v->nb[2]; ++ vp = ggml_permute(ctx0, v, 0, 2, 1, 3); ++ if (v_trans) { ++ vp = ggml_transpose(ctx0, vp); ++ } ++ if (vp->type == GGML_TYPE_F32) { ++ vp = ggml_cast(ctx0, vp, GGML_TYPE_F16); ++ } + } + ggml_tensor * kp = ggml_permute(ctx0, k_dca, 0, 2, 1, 3); + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -788,6 +788,17 @@ + const int64_t n_batch = Q->ne[3]; + const int64_t n_rows = n_head * n_q * n_batch; + GGML_ASSERT(dst->ne[0] == DV + 1 && "flash_attn_ext_lse dst must pack [O|lse] on ne[0]"); ++ ++ // opencoti F5 dca (DCA all-KV C-perf, bug-720): the FUSED regime (op_params[4]==3) self-contains its ++ // attention + normalization and now dequant-on-lifts a scalar-quantized K/V INSIDE its own launch ++ // (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case). It never runs the streaming_lse recompute below, so ++ // route it out BEFORE the recompute-only f32-Q / f16-K asserts (the streaming_lse_kernel reads K as f16, ++ // hence those gates). Without this early-out a quantized DCA cache aborts here at line ~792. ++ if (ggml_get_op_params_i32(dst, 4) == 3) { ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst); ++ return; ++ } ++ + GGML_ASSERT(Q->type == GGML_TYPE_F32 && "flash_attn_ext_lse expects f32 Q for the lse recompute"); + GGML_ASSERT(K->type == GGML_TYPE_F16 && "flash_attn_ext_lse expects f16 K for the lse recompute"); + +@@ -820,19 +831,11 @@ + const int dca_chunk = ggml_get_op_params_i32(dst, 3); + const int dca_regime = ggml_get_op_params_i32(dst, 4); + +- // opencoti F5 dca perf (T87.pD Branch B): FUSED single-pass DCA. The three band passes (INTER/SUCC/ +- // INTRA) are folded into ONE block-per-tile kernel that streams K/V once and swaps the pre-roped Q +- // variant at the 2 phase boundaries — no per-regime LSE emit, no host combine, no streaming_lse +- // fallback. The fused case reads its inputs straight off dst (src[0]=Q_intra, src[1]=K, src[2]=V, +- // src[3]=mask_intra, src[4]=pos_q_real, src[5]=Q_succ, src[6]=Q_inter, op_params[3]=chunk) and writes +- // normalized O into the leading DV*n_rows of dst's [O|lse] buffer (the trailing lse row is left +- // untouched — dca.cpp slices it off). Bands partition the causal range [0,i] ⇒ one online-softmax +- // pass is bit-faithful to the 3-op + host LSE combine it replaces. +- if (dca_regime == 3) { +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst); +- return; +- } +- ++ // opencoti F5 dca perf (T87.pD Branch B): the FUSED single-pass DCA regime (op_params[4]==3) — three ++ // band passes (INTER/SUCC/INTRA) folded into ONE block-per-tile kernel, no per-regime LSE emit / host ++ // combine / streaming_lse fallback — is dispatched at the TOP of this function (bug-720), before the ++ // f32-Q / f16-K recompute asserts, so a quantized DCA cache reaches its in-launch dequant. Only the ++ // non-fused (regime 0/1/2) lse-recompute path continues below. + float * lse_out = (float *) dst->data + DV * n_rows; + opencoti_fattn_dst_lse = lse_out; + opencoti_fattn_dst_lse_written = false; +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -2360,7 +2360,14 @@ + const ggml_tensor * Qi = dst->src[6]; // Q_inter + const ggml_tensor * Ms = dst->src[7]; // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ fast path) + const ggml_tensor * Mi = dst->src[8]; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ fast path) +- GGML_ASSERT(K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && "dca fused: f16 K/V required"); ++ // opencoti F5 dca (DCA all-KV C-perf, bug-720): the fused kernel reads f16 K/V, but a scalar-quantized ++ // cache is now dequant'd to f16 IN THIS LAUNCH below (single-pass to_fp16_nc into a transient pool ++ // buffer, mirroring launch_fattn) instead of being whole-cache materialized to f16 in the graph every ++ // forward (the prefill cliff: ~138 vs ~1212 tps at 98k/q5_0). Accept f16, or any type with a to_fp16_nc ++ // converter; reject anything else (the kernel would have no f16 source). ++ GGML_ASSERT((K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) && ++ (V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) && ++ "dca fused: K/V must be f16 or have a to_fp16_nc converter"); + GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors"); + // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER + // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent. +@@ -2410,6 +2417,44 @@ + ggml_cuda_pool & pool = ctx.pool(); + cudaStream_t main_stream = ctx.stream(); + ++ // opencoti F5 dca (DCA all-KV C-perf, bug-720): dequant-on-lift a scalar-quantized K/V into a transient ++ // f16 pool buffer here — the SAME single-pass to_fp16_nc convert the stock MMA path uses (launch_fattn, ++ // fattn-common.cuh) — replacing the old two-stage whole-cache graph lift (dca.cpp dca_lift_to_f16, ++ // q->f32->f16) that ran every forward and dominated quant-KV DCA-on prefill. K/V are PERMUTED views ++ // (kp/vp, dim-0 = the contiguous block axis), so use the non-contiguous (strided) converter and pack to ++ // a contiguous f16 slot; the kernel then reads the slot with contiguous strides. f16 K/V skip the ++ // convert (k_data/v_data alias the tensor, strides unchanged ⇒ byte-identical to the prior f16 path). ++ ggml_cuda_pool_alloc K_f16(pool); ++ ggml_cuda_pool_alloc V_f16(pool); ++ const char * k_data = (const char *) K->data; ++ size_t nb_k1 = K->nb[1], nb_k2 = K->nb[2], nb_k3 = K->nb[3]; ++ const char * v_data = (const char *) V->data; ++ size_t nb_v1 = V->nb[1], nb_v2 = V->nb[2], nb_v3 = V->nb[3]; ++ if (K->type != GGML_TYPE_F16) { ++ const size_t ts = ggml_type_size(K->type); ++ GGML_ASSERT(K->nb[0] == ts && "dca fused: quant K must have contiguous dim-0 (block axis)"); ++ to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(K->type); ++ K_f16.alloc(ggml_nelements(K)); ++ to_fp16(k_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], ++ (int64_t) (nb_k1 / ts), (int64_t) (nb_k2 / ts), (int64_t) (nb_k3 / ts), main_stream); ++ k_data = (const char *) K_f16.ptr; ++ nb_k1 = K->ne[0] * sizeof(half); ++ nb_k2 = K->ne[1] * nb_k1; ++ nb_k3 = K->ne[2] * nb_k2; ++ } ++ if (V->type != GGML_TYPE_F16) { ++ const size_t ts = ggml_type_size(V->type); ++ GGML_ASSERT(V->nb[0] == ts && "dca fused: quant V must have contiguous dim-0 (block axis)"); ++ to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(V->type); ++ V_f16.alloc(ggml_nelements(V)); ++ to_fp16(v_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], ++ (int64_t) (nb_v1 / ts), (int64_t) (nb_v2 / ts), (int64_t) (nb_v3 / ts), main_stream); ++ v_data = (const char *) V_f16.ptr; ++ nb_v1 = V->ne[0] * sizeof(half); ++ nb_v2 = V->ne[1] * nb_v1; ++ nb_v3 = V->ne[2] * nb_v2; ++ } ++ + const int ntiles_x = (Q->ne[1] + ncols1 - 1) / ncols1; + const int gqa_ratio = Q->ne[2] / K->ne[2]; + const int ntiles_z_gqa = (gqa_ratio + ncols2 - 1) / ncols2; +@@ -2490,7 +2535,7 @@ + + const dim3 grid(nblocks, 1, 1); + kernel<<>>( +- (const char *) Q->data, (const char *) K->data, (const char *) V->data, ++ (const char *) Q->data, k_data, v_data, // opencoti dca bug-720: f16 slot (or aliased f16 tensor) + (const char *) mask->data, + Ms ? (const char *) Ms->data : nullptr, // opencoti F5 dca (#617B): dense SUCC mask (null ⇒ fast path) + Mi ? (const char *) Mi->data : nullptr, // opencoti F5 dca (#617B): dense INTER mask (null ⇒ fast path) +@@ -2499,8 +2544,8 @@ + (float *) KQV->data, dst_meta_ptr, + scale, 0.0f /*max_bias*/, 1.0f /*m0*/, 1.0f /*m1*/, n_head_log2, logit_softcap, + Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], +- K->ne[0], K->ne[1], K->ne[2], K->ne[3], K->nb[1], K->nb[2], K->nb[3], +- V->nb[1], V->nb[2], V->nb[3], ++ K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb_k1, nb_k2, nb_k3, // opencoti dca bug-720: slot strides ++ nb_v1, nb_v2, nb_v3, + mask->ne[1], mask->ne[2], mask->ne[3], mask->nb[1], mask->nb[2], mask->nb[3]); + CUDA_CHECK(cudaGetLastError()); + diff --git a/patches/0086-f16-moe-mmf-prefill.patch b/patches/0086-f16-moe-mmf-prefill.patch new file mode 100644 index 0000000000000000000000000000000000000000..c874af0ec9ee94c3e3cd604df049a0cd2886c41e --- /dev/null +++ b/patches/0086-f16-moe-mmf-prefill.patch @@ -0,0 +1,49 @@ +From: opencoti +Subject: [PATCH 0086] F16 MoE prefill — route large-batch f16 mul_mat_id to MMF (#555) + +ggml_cuda_mul_mat_id had no fast CUDA path for f16/bf16/f32 experts at prefill batch: +MMQ is quantized-only, and ggml_cuda_should_use_mmf rejected mul_mat_id whenever +src1_ncols (= n_tokens) exceeded 512 (or 128 when the expert dim > 1024). At prefill +(~32k tokens) that gate always failed, so f16 MoE fell into the host-side sorted +fallback in ggml_cuda_mul_mat_id: a per-call cudaStreamSynchronize (which also disables +CUDA graphs) + an O(n_expert*n_tokens*n_used) host sort + a per-expert cuBLAS GEMM loop. +Quantized MoE dodges all of it via the single fused MMQ-id kernel. Net: f16 128e prefill +ran ~12x slower than Q6_K (690 vs 8857 tok/s on Gemma-4-A4B 128e). + +The MMF kernel's `ncols_dst > 16` branch (ggml_cuda_mul_mat_f, this file) already performs +GPU-side expert compaction (ggml_cuda_launch_mm_ids_helper, mmid.cuh) entirely on-stream +and handles arbitrary n_tokens -- the gate was a conservative tuning heuristic, not a +capability limit. This compiles out that mul_mat_id gate so large-batch f16/bf16/f32 MoE +rides the fast GPU-compacted MMF path. Measured on a real 128e-F16 (bs2 RTX PRO 6000): +prefill 690 -> 3081 tok/s (4.47x); RULER-vt@32k retrieval 100.0 (== Q6_K, correctness-clean). +The residual gap to Q6_K is just the legitimate f16-vs-q6 weight-bandwidth ratio (~2.2x). +Quantized MoE (MMQ path) and dense f16 mul_mat are untouched -- both are rejected/handled +before this gate. The upstream gate is preserved, compiled out, behind +-DOPENCOTI_MMF_LEGACY_GATE. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/mmf.cu b/llama.cpp/ggml/src/ggml-cuda/mmf.cu +--- a/llama.cpp/ggml/src/ggml-cuda/mmf.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/mmf.cu +@@ -160,11 +160,22 @@ + } + + if (mul_mat_id) { ++ // opencoti-hook: f16-moe-prefill (#555) — the upstream heuristic below rejected ++ // large-batch (prefill) f16/bf16/f32 mul_mat_id, sending it to the host-sync sorted ++ // fallback in ggml_cuda_mul_mat_id (per-call cudaStreamSynchronize + per-expert cuBLAS ++ // loop + disabled CUDA graphs => measured 12x slower than the quantized MMQ-id path: ++ // 690 vs 8857 tok/s prefill on Gemma-4-A4B 128e). The MMF kernel's `ncols_dst > 16` ++ // branch already performs GPU-side expert compaction (ggml_cuda_launch_mm_ids_helper, ++ // mmid.cuh) entirely on-stream and handles arbitrary n_tokens, so keep f16 MoE on the ++ // fast fused MMF path for any batch size. Upstream gate preserved (disabled) for ref; ++ // re-enable with -DOPENCOTI_MMF_LEGACY_GATE. ++#ifdef OPENCOTI_MMF_LEGACY_GATE + if (src0_ne[1] <= 1024 && src1_ncols > 512) { + return false; + } else if(src0_ne[1] > 1024 && src1_ncols > 128) { + return false; + } ++#endif + } else { + if (GGML_CUDA_CC_IS_RDNA3_0(cc) && src1_ncols > 8) { + return false; diff --git a/patches/0087-dca-yarn-factor.patch b/patches/0087-dca-yarn-factor.patch new file mode 100644 index 0000000000000000000000000000000000000000..b4b96cd4a57f58e57d739b277010b22461068650 --- /dev/null +++ b/patches/0087-dca-yarn-factor.patch @@ -0,0 +1,56 @@ +From: opencoti +Subject: [PATCH 0087] DCA: wire --dca-yarn-factor into build_dca_rope mscale (bug-740 / #550) + +--dca-yarn-factor was fully plumbed (arg.cpp -> params -> cparams.dca_yarn_factor, +public C API, TS adapter) but never READ by the rope path, so the flag was inert +(bug-740). build_dca_rope now composes it into the attention scaling: + dca_attn_factor = attn_factor * cparams.dca_yarn_factor +and passes that to both ggml_rope_multi (MROPE) and ggml_rope_ext (NEOX) in place of +the bare attn_factor. This is the YaRN mscale piece of the DCA+YaRN composition: DCA +already keeps q-k distances in-distribution via the per-regime chunk remap, so only the +softmax temperature (mscale) needs stretching for ctx past the chunk grid -- freq_scale +/ freq_base stay the model's baked values. Host-only (graph construction); the existing +CUDA rope kernel reads attn_factor from op_params unchanged, so no DSO rebuild. + +Default 1.0 => dca_attn_factor == attn_factor exactly (x1.0f is bit-identity for finite +floats), so the flag-unset path is byte-identical to upstream -- the fix only makes a +previously-inert flag live; it cannot regress default behavior. + +Empirical characterization (Qwen3-4B, native 40960, DCA chunk 16384, RULER 5-needle +substring under ignore_eos; NOT greedy-first-token): + f16 KV @ 2.4x native: yarn 1.0->5/5 1.25->4/5 1.5->1/5 2.0->0/5 (monotonic: flag live) + q8_0 KV ladder yarn=1.0: 2.4x->4 4.0x->5 5.0x->3 6.0x->4 (plain-DCA holds deep) + q8_0 KV @ 6.0x native: yarn 1.0->4 1.1->4 1.25->4 (flat; 1.18 = principled mscale, no win) +Verdict: the mscale lever has NO beneficial regime for DCA retrieval -- it only +over-sharpens already-concentrated attention. DEFAULT 1.0 IS CORRECT. Ships as a +power-user knob (raising it degrades retrieval); documented in docs/evaluations/context.md. + +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -164,15 +164,23 @@ + // Rope the DCA per-regime Q (or insert-K) faithfully for the arch. `pos` must be the width + // build_attn_inp_dca allocated: 4*n_tokens for MROPE (section split below), n_tokens for NEOX. + ggml_tensor * llm_graph_context::build_dca_rope(ggml_tensor * x, ggml_tensor * pos, const dca_rope & rp) const { ++ // opencoti-hook: dca-yarn-factor (bug-740 / #550) — compose a YaRN mscale stretch with DCA. ++ // cparams.dca_yarn_factor (1.0 = none; set by --dca-yarn-factor) multiplies the attention scaling ++ // (mscale / attn_factor) so the extending-regime softmax is re-tempered for ctx past the chunk grid ++ // (the final 2-4x of the DCA+YaRN sub-native extension recipe; see docs/features/gemma4_dca.md). DCA ++ // already keeps q-k distances in-distribution via the per-regime chunk remap, so only the mscale needs ++ // stretching — freq_scale/freq_base stay the model's baked values. Default 1.0 => this rope call is ++ // byte-identical to upstream (inert unless the flag is set). ++ const float dca_attn_factor = attn_factor * cparams.dca_yarn_factor; + if (dca_is_mrope(rope_type)) { + int sections[GGML_MROPE_SECTIONS] = { rp.sections[0], rp.sections[1], rp.sections[2], rp.sections[3] }; + return ggml_rope_multi(ctx0, x, pos, rp.freq_factors, + rp.n_rot, sections, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow); ++ ext_factor, dca_attn_factor, beta_fast, beta_slow); + } + return ggml_rope_ext(ctx0, x, pos, rp.freq_factors, + rp.n_rot, rope_type, n_ctx_orig, rp.freq_base, rp.freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow); ++ ext_factor, dca_attn_factor, beta_fast, beta_slow); + } + + // opencoti F5 dca #444 (DCA all-KV C1): dequant-on-lift a cached K/V tensor to f16 for the f16-only diff --git a/patches/0088-sparsev-autopolicy.patch b/patches/0088-sparsev-autopolicy.patch new file mode 100644 index 0000000000000000000000000000000000000000..6b32155690b875e4cc73b4ce39af9b47442c94f5 --- /dev/null +++ b/patches/0088-sparsev-autopolicy.patch @@ -0,0 +1,63 @@ +From: opencoti +Subject: [PATCH 0088] sparse-v auto-policy: self-configure quantized-V skip from model arch (#565) + +The FA-VEC quantized-V skip (sparse-V, #546) is a MEASURED lossless decode win on +interleaved-SWA models (Gemma-4: the small, concentrated global V-cache lets a +peak-relative static tau=0.05 harvest the sub-mean V tail — RULER niah 100 at +64k/128k/256k on A4B-62e AND official Gemma-4-31B), but a no-win on full-attention +models (Qwen: diffuse weights cluster near the mean, nothing safely skippable). The +kernel reads its threshold from the TURBO_SPARSE_V_TAU env (ggml-cuda is a runtime +DSO, so env is the cross-DSO channel). Until now a user had to know to set that env. + +This puts the intelligence in the BINARY so a STANDALONE `llamafile --server` self- +configures: at llama_context construction, when the user has NOT set TURBO_SPARSE_V_TAU +and the model is iSWA (hparams.swa_type != NONE) with a quantized V cache, default the +env to 0.05 (overwrite=0 — an explicit env still wins). Full-attention models leave it +unset -> kernel tau=0 -> dense, byte-identical. f16/bf16 V is inert via the kernel's own +`if constexpr type_V != F16/BF16` guard. Host-only (no DSO change); the existing kernel +getenv reads the pre-populated value. Announced once at WARN (the llamafile server +suppresses the model-load INFO block, and a binary that auto-changes inference should +say so + tell the user how to override). + +Verified (3090): decision table PASS (policy fires iff iSWA+quant-V+no-env: A4B q8_0 +auto=engaged, override/f16/Qwen-full-attn=off) + A/B at 24k lossless + non-slower. +--- +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 722bd7f..2d87414 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -54,6 +54,34 @@ llama_context::llama_context( + + const auto & hparams = model.hparams; + ++ // opencoti-hook: sparse-v auto-policy (#565) — see docs/features/sparse_attn.md ++ // Self-configure the FA-VEC quantized-V skip (#546) from the model's OWN architecture, so a ++ // STANDALONE `llamafile --server` gets the win with no flags (the intelligence lives in the binary, ++ // not the opencoti TS adapter). The skip is a MEASURED lossless decode win on interleaved-SWA models ++ // (Gemma-4: the tiny, concentrated global V-cache lets a peak-relative static τ=0.05 harvest the ++ // sub-mean V tail — niah 100 at 64k/128k/256k on A4B-62e AND official Gemma-4-31B), but a no-win on ++ // full-attention models (Qwen: diffuse weights cluster near the mean, nothing safely skippable). It ++ // only engages for a QUANTIZED V cache (the kernel's `if constexpr type_V != F16/BF16` guard; -ctv ++ // f16 is inert). ggml-cuda is a runtime-loaded DSO, so the kernel reads its threshold from the ++ // TURBO_SPARSE_V_TAU env (the existing cross-DSO channel) — we just pre-populate a smart default when ++ // the user has not, with overwrite=0 so an explicit TURBO_SPARSE_V_TAU (or eps/sink/recent) still ++ // wins. Set as early as possible (before any graph_reserve / decode) so the kernel's lazy getenv ++ // caches the policy value. Full-attention models leave the env unset → kernel τ=0 → dense, byte-identical. ++ if (!hparams.vocab_only && getenv("TURBO_SPARSE_V_TAU") == nullptr) { ++ const bool is_iswa = hparams.swa_type != LLAMA_SWA_TYPE_NONE; ++ // Mirror the kernel's engage guard exactly: any non-float V cache (q8_0/q4_0/turbo*/tcq) is eligible. ++ const bool quant_v = params.type_v != GGML_TYPE_F16 && params.type_v != GGML_TYPE_BF16 && ++ params.type_v != GGML_TYPE_F32; ++ if (is_iswa && quant_v) { ++ setenv("TURBO_SPARSE_V_TAU", "0.05", /*overwrite=*/0); ++ // WARN level (not INFO): the llamafile server suppresses the model-load INFO block, and a binary ++ // that auto-changes inference behavior should announce it audibly + tell the user how to override. ++ LLAMA_LOG_WARN("%s: sparse-v auto-policy: iSWA + quantized V (%s) -> TURBO_SPARSE_V_TAU=0.05 " ++ "(lossless decode win; export TURBO_SPARSE_V_TAU to override)\n", ++ __func__, ggml_type_name(params.type_v)); ++ } ++ } ++ + cparams.n_seq_max = std::max(1u, params.n_seq_max); + if (cparams.n_seq_max > LLAMA_MAX_SEQ) { + throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ)); diff --git a/patches/0089-sparse-attn-vslash-foundation.patch b/patches/0089-sparse-attn-vslash-foundation.patch new file mode 100644 index 0000000000000000000000000000000000000000..b7ab3bd0096f88dc077daec74996d7fd8578cbec --- /dev/null +++ b/patches/0089-sparse-attn-vslash-foundation.patch @@ -0,0 +1,2910 @@ +From: opencoti +Subject: [PATCH 0089] sparse-attn vslash + uncaptured foundation (#551/#557-568, #591/#592) + +Consolidated capture of tree work that predated the 0090 (compute-reserve) +snapshot but was never emitted as a patch: the sparse-attention vertical-slash +feature (ggml_sparse_attn_vslash op, sparse-attn.cu/.cuh estimate+block-mass +kernels, llama-graph wiring, sparse_attn_block_size plumbed through the KV-cache +ctors) plus the #591/#592 decode-overhead work (async input H2D upload, on-device +greedy draft argmax) and assorted fattn/turbo/tcq touch-ups from that window. +Reconstructed as diff(chain-0088 baseline, vendor-backup 20260701-123044) after +bug-2108 (see buglog) showed 0090+ were captured against this uncaptured state +and the committed chain no longer strict-applied from a clean 0.10.3 tree. + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +index cfd3b29..89bd9dd 100644 +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1539,6 +1539,72 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.dca_yarn_factor = std::stof(value); + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DCA_YARN_FACTOR")); ++ // opencoti #551 sparse-attn — Quest-style block-selector sparse attention — see docs/features/sparse_attn.md ++ add_opt(common_arg( ++ {"--sparse-attn"}, "MODE", ++ "Block-selector sparse attention for long-context decode (Quest-style, training-free): off (default), on. " ++ "Per-query, keeps only the top-K KV blocks by an exact min/max upper bound (+ recent/sink window) and " ++ "skips the rest before the QK dot. Composes with --dca and quantized KV. Default off = byte-identical.", ++ [](common_params & params, const std::string & value) { ++ if (value == "off" || value == "false" || value == "0") params.sparse_attn_enabled = false; ++ else if (value == "on" || value == "true" || value == "1") params.sparse_attn_enabled = true; ++ else throw std::invalid_argument("--sparse-attn must be on|off"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN")); ++ add_opt(common_arg( ++ {"--sparse-attn-block-size"}, "N", ++ "Sparse-attn KV-block granularity B_SEL in tokens (default 64). Min/max key bounds are kept per block; " ++ "align to a multiple of the quant block size (q5_0/q4_0 blck=32) for clean skipping.", ++ [](common_params & params, const std::string & value) { ++ params.sparse_attn_block_size = (uint32_t) std::stoul(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_BLOCK_SIZE")); ++ add_opt(common_arg( ++ {"--sparse-attn-topk"}, "N", ++ "Sparse-attn: number of KV blocks to keep per query by upper-bound score (0 = all blocks, i.e. " ++ "off; 'auto' = adaptive ¼ of blocks, floor 64).", ++ [](common_params & params, const std::string & value) { ++ if (value == "auto") params.sparse_attn_topk = 0xFFFFFFFFu; // adaptive: max(64, n_blocks/4) ++ else params.sparse_attn_topk = (uint32_t) std::stoul(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_TOPK")); ++ add_opt(common_arg( ++ {"--sparse-attn-recent"}, "N", ++ "Sparse-attn: always-keep recent window in tokens (streaming-LLM safety; 0 = none).", ++ [](common_params & params, const std::string & value) { ++ params.sparse_attn_recent = (uint32_t) std::stoul(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_RECENT")); ++ add_opt(common_arg( ++ {"--sparse-attn-sink"}, "N", ++ "Sparse-attn: number of leading 'sink' KV blocks always kept (attention-sink safety; default 1).", ++ [](common_params & params, const std::string & value) { ++ params.sparse_attn_sink = (uint32_t) std::stoul(value); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_SINK")); ++ add_opt(common_arg( ++ {"--sparse-attn-refresh"}, "N", ++ "Sparse-attn PERF: re-run the block selector every N decode tokens and reuse the cached selection in " ++ "between (amortizes the per-step selector cost; default 8, 1 = every step). The sink block is always " ++ "kept so a reused selection never empties.", ++ [](common_params & params, const std::string & value) { ++ params.sparse_attn_refresh = (uint32_t) std::stoul(value); ++ if (params.sparse_attn_refresh < 1) params.sparse_attn_refresh = 1; ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_REFRESH")); ++ add_opt(common_arg( ++ // opencoti-hook: sparse-attn-vslash — #551 selector mechanism. ++ {"--sparse-attn-mode"}, "MODE", ++ "Sparse-attn block-selector mechanism: 'quest' (default) = geometric per-dim kmin/kmax spread bound; " ++ "'vslash' = vertical-slash content-driven selector (per-block max attention logit over the recent " ++ "query window — picks the blocks the query actually attends to). vslash succeeds at long context " ++ "where quest's lossless fraction rises with ctx. Only meaningful with --sparse-attn on.", ++ [](common_params & params, const std::string & value) { ++ if (value == "quest" || value == "0") params.sparse_attn_mode = 0; ++ else if (value == "vslash" || value == "1") params.sparse_attn_mode = 1; ++ else throw std::invalid_argument("--sparse-attn-mode must be quest|vslash"); ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPARSE_ATTN_MODE")); + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--pcie-autodetect"}, "MODE", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +index 7503291..190682c 100644 +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1649,6 +1649,14 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.dca_enabled = params.dca_enabled; + cparams.dca_chunk_size = params.dca_chunk_size; + cparams.dca_yarn_factor = params.dca_yarn_factor; ++ // opencoti #551 sparse-attn — see docs/features/sparse_attn.md ++ cparams.sparse_attn_enabled = params.sparse_attn_enabled; ++ cparams.sparse_attn_block_size = params.sparse_attn_block_size; ++ cparams.sparse_attn_topk = params.sparse_attn_topk; ++ cparams.sparse_attn_recent = params.sparse_attn_recent; ++ cparams.sparse_attn_sink = params.sparse_attn_sink; ++ cparams.sparse_attn_refresh = params.sparse_attn_refresh; // opencoti #551 PERF (was omitted — flag was inert) ++ cparams.sparse_attn_mode = params.sparse_attn_mode; // opencoti-hook: sparse-attn-vslash — #551 + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index 205ec67..b688492 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -469,6 +469,19 @@ struct common_params { + bool dca_enabled = false; + uint32_t dca_chunk_size = 0; + float dca_yarn_factor = 1.0f; ++ // opencoti #551 sparse-attn — Quest-style block-selector sparse attention (docs/features/sparse_attn.md) ++ // off by default; block_size 128 (B_SEL == FA-VEC tile width nthreads; clamped to a multiple of 128 in ++ // llama-graph so the compacted-list consumer's tiles align to blocks); topk 0 = all blocks (off), ++ // 0xFFFFFFFF = auto ¼; recent 0 = no forced window (tokens); sink = leading sink blocks always kept (1). ++ bool sparse_attn_enabled = false; ++ uint32_t sparse_attn_block_size = 128; ++ uint32_t sparse_attn_topk = 0; ++ uint32_t sparse_attn_recent = 0; ++ uint32_t sparse_attn_sink = 1; ++ uint32_t sparse_attn_refresh = 8; // #551 PERF: re-run the block selector every N decode tokens (amortize; 1 = every step) ++ // opencoti-hook: sparse-attn-vslash — #551 selector mechanism: 0 = Quest geometric kmin/kmax bound ++ // (default, byte-identical to prior behavior); 1 = vertical-slash content-driven max-logit selector. ++ uint32_t sparse_attn_mode = 0; + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +index 3c72dd4..614da7f 100644 +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -612,6 +612,10 @@ extern "C" { + + GGML_OP_FLASH_ATTN_EXT_LSE, // opencoti F5 dca — #593 (O+lse packed) + ++ GGML_OP_SPARSE_ATTN_FILL, // opencoti-hook: sparse-attn — #551 Quest per-block kmin/kmax fill ++ GGML_OP_SPARSE_ATTN_SELECT, // opencoti-hook: sparse-attn — #551 Quest per-(seq,qtile) block bitmask ++ GGML_OP_SPARSE_ATTN_VSLASH, // opencoti-hook: sparse-attn-vslash — #551 content-driven block selector ++ + GGML_OP_COUNT, + }; + +@@ -2577,6 +2581,67 @@ extern "C" { + int group_size, + struct ggml_tensor * scale); + ++ // opencoti-hook: sparse-attn — #551 Quest block-selector. Reduce the freshly-written ++ // K (pre-quant f16) into per-block channel-wise min/max, merged in-place into the ++ // kbounds side-cache. kbounds: [n_embd_k, n_block, 2] f16 ([..,0]=min, [..,1]=max); ++ // the op writes into kbounds' buffer (ggml_cpy idiom) so it carries the write edge to ++ // the selector. k_cur = new tokens' K; k_idxs = their cache positions. block_size = B_SEL. ++ // See docs/features/sparse_attn.md (S2/S3 design). ++ GGML_API struct ggml_tensor * ggml_sparse_attn_fill( ++ struct ggml_context * ctx, ++ struct ggml_tensor * kbounds, ++ struct ggml_tensor * k_cur, ++ struct ggml_tensor * k_idxs, ++ int block_size); ++ ++ // opencoti-hook: sparse-attn — #551 Quest block selector. Emits a SORTED, −1-padded compacted ++ // list of selected block indices per sequence (union over KV-heads): block b is kept iff any ++ // KV-head's Quest upper bound for b is in the top-K, OR b is in the recent/sink window. The ++ // FA-VEC consumer walks the list and stops at the first −1 (O(selected), not O(capacity)). ++ // ++ // #551 PERF (amortized selection): when block_sel != nullptr the op writes IN-PLACE into that ++ // persistent per-stream buffer ([n_block+1, n_seq] I32) and the kernel re-runs the selection ++ // ONLY every `refresh` decode tokens — reading the decode position from k_idxs[0] on the device ++ // so it stays correct under CUDA-graph replay — reusing the cached list on the in-between steps. ++ // The sink block is always kept, so a reused (stale) list is never empty and never OOB (the ++ // FA-VEC loop skips block starts >= n_kv). block_sel == nullptr ⇒ legacy transient per-step list ++ // (dst is a fresh [n_block, n_seq]; k_idxs/refresh ignored). op_params = ++ // {block_size, n_block, k_sel, recent_blocks, sink_blocks, refresh}. ++ GGML_API struct ggml_tensor * ggml_sparse_attn_select( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, // rotated FA query [n_embd_k, n_head, n_qtile, n_seq] ++ struct ggml_tensor * kbounds, // [n_embd_k, n_block, 2] from the fill op ++ struct ggml_tensor * block_sel, // persistent [n_block+1, n_seq] I32 write target, or NULL (transient) ++ struct ggml_tensor * k_idxs, // KV write indices — decode position source (NULL when block_sel NULL) ++ int block_size, ++ int n_block, ++ int k_sel, ++ int recent_blocks, ++ int sink_blocks, ++ int refresh, // re-select every N decode tokens (>=1; ignored when block_sel NULL) ++ int sub_per_block); // #551: kbounds sub-blocks per walk-block (finer Quest bound); 1 = none ++ ++ // opencoti-hook: sparse-attn-vslash — #551 content-driven block selector (MInference-style). Scores ++ // each block by the EXACT per-block max attention logit over the last `q_win` query rows ++ // (score(b) = max over (q in window, key in block b, head) of q·key), instead of the Quest geometric ++ // kmin/kmax spread bound (which over-ranks high-spread filler and degrades with context). Reuses the ++ // select kernel's top-K + recent/sink sorted-compaction emit verbatim; src[1] is the actual cache K ++ // (f16), not a kbounds side-cache. block_sel/k_idxs/refresh semantics identical to ggml_sparse_attn_select. ++ // op_params = {block_size, n_block, k_sel, recent_blocks, sink_blocks, refresh, q_win}. ++ GGML_API struct ggml_tensor * ggml_sparse_attn_vslash( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, // rotated FA query [n_embd_k, n_head, n_q, n_seq] (f32) ++ struct ggml_tensor * k_full, // cache K [head_dim, n_head_kv, n_kv, 1] (f16) ++ struct ggml_tensor * block_sel, // persistent [n_block+1, n_seq] I32 write target, or NULL (transient) ++ struct ggml_tensor * k_idxs, // KV write indices — decode position source (NULL when block_sel NULL) ++ int block_size, ++ int n_block, ++ int k_sel, ++ int recent_blocks, ++ int sink_blocks, ++ int refresh, // re-select every N decode tokens (>=1; ignored when block_sel NULL) ++ int q_win); // # of recent query rows that define the importance window (>=1) ++ + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + +diff --git a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +index 468f762..697bf8d 100644 +--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c ++++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +@@ -2124,6 +2124,9 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm + // owns the FFN nodes, so the CPU scheduler never reaches here. + GGML_ABORT("MOE_FUSED_UP_GATE has no CPU implementation (CUDA-only op)"); + } break; ++ case GGML_OP_SPARSE_ATTN_FILL: // opencoti-hook: sparse-attn — #551 (CUDA-only, shares abort) ++ case GGML_OP_SPARSE_ATTN_SELECT: // opencoti-hook: sparse-attn — #551 (CUDA-only, shares abort) ++ case GGML_OP_SPARSE_ATTN_VSLASH: // opencoti-hook: sparse-attn-vslash — #551 (CUDA-only, shares abort) + case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 (CUDA-only, shares abort) + case GGML_OP_STREAMING_FLASH_ATTN: + { +@@ -2417,6 +2420,9 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { + // scheduled on CPU. Single task keeps the planner well-formed. + n_tasks = 1; + } break; ++ case GGML_OP_SPARSE_ATTN_FILL: // opencoti-hook: sparse-attn — #551 (CUDA-only) ++ case GGML_OP_SPARSE_ATTN_SELECT: // opencoti-hook: sparse-attn — #551 (CUDA-only) ++ case GGML_OP_SPARSE_ATTN_VSLASH: // opencoti-hook: sparse-attn-vslash — #551 (CUDA-only) + case GGML_OP_FLASH_ATTN_EXT_LSE: // opencoti F5 dca — #593 (CUDA-only) + case GGML_OP_STREAMING_FLASH_ATTN: + { +diff --git a/llama.cpp/ggml/src/ggml-cuda/cpy.cu b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +index e769b7b..7aa2140 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/cpy.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/cpy.cu +@@ -602,7 +602,10 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg + // set-rows.cu) is fetched by the cast's row width (ne00 = n_embd_gqa); nullptr ⇒ identity + // (default-off / no slot). See docs/features/poly_kv.md (M6-S2 perf-path Level A). + } else if ((src0->type == GGML_TYPE_TURBO2_0 || src0->type == GGML_TYPE_TURBO3_0 || +- src0->type == GGML_TYPE_TURBO4_0 || src0->type == GGML_TYPE_TURBO8_0) && ++ src0->type == GGML_TYPE_TURBO4_0 || src0->type == GGML_TYPE_TURBO8_0 || ++ // opencoti-hook #533: TCQ→f16 dequant-on-lift (prefill-MMA + DCA-compose). The TCQ ++ // kernels ignore d_scale_inv (no InnerQ) — scale_inv=nullptr handled inside the kernel. ++ src0->type == GGML_TYPE_TURBO3_TCQ || src0->type == GGML_TYPE_TURBO2_TCQ) && + src1->type == GGML_TYPE_F16) { + const float * d_scale_inv = nullptr; + int head_dim = 0; +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +index e5181d7..26cd885 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -26,6 +26,9 @@ typedef void (* fattn_kernel_t)( + const char * __restrict__ sinks, + const int * __restrict__ KV_max, + const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim ++ const int * __restrict__ block_sel, // opencoti-hook: sparse-attn #551 — sorted −1-padded selected-block list (nullptr=dense) ++ const int32_t sparse_B, // opencoti-hook: sparse-attn #551 — block size (multiple of nthreads) ++ const int32_t sparse_nwords, // opencoti-hook: sparse-attn #551 — list stride == n_block per sequence + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -59,6 +62,17 @@ extern thread_local const int * opencoti_fattn_dca_pos_q; + extern thread_local int opencoti_fattn_dca_chunk; + extern thread_local int opencoti_fattn_dca_regime; + ++// opencoti-hook: sparse-attn-vslash side-output (#551 S1) — consume-once side channel for emitting the ++// per-block question-window attention MASS as a FREE byproduct of the prefill MMA FA (the softmax is ++// computed anyway; #551 vslash separate-pass died on prefill cost it couldn't amortize — Quest wins by ++// computing its bound for free, so vslash must too). The vslash graph site sets this immediately before ++// the prefill FA dispatch (mode==1, n_q>1); launch_fattn drains it. The MMA kernel (S2) accumulates ++// per-block window mass into block_mass when non-null. nullptr = byte-identical legacy path. Def in fattn.cu. ++extern thread_local float * opencoti_fattn_block_mass; // [n_block] out, PRE-ZEROED by the caller ++extern thread_local int opencoti_fattn_block_mass_nblock; ++extern thread_local int opencoti_fattn_block_mass_B; // keys per block ++extern thread_local int opencoti_fattn_block_mass_qwin; // # of trailing query rows in the window ++ + typedef float (*vec_dot_KQ_t)( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds); + +@@ -1187,6 +1201,19 @@ void launch_fattn( + } + opencoti_fattn_dst_lse = nullptr; + ++ // opencoti-hook: sparse-attn-vslash side-output (#551 S1) — drain the consume-once block-mass channel. ++ // S1 only drains (no kernel consumer yet) so this is byte-identical; S2 threads these into the kernel. ++ float * block_mass = opencoti_fattn_block_mass; ++ const int block_mass_nblock = opencoti_fattn_block_mass_nblock; ++ const int block_mass_B = opencoti_fattn_block_mass_B; ++ const int block_mass_qwin = opencoti_fattn_block_mass_qwin; ++ opencoti_fattn_block_mass = nullptr; ++ opencoti_fattn_block_mass_nblock = 0; ++ opencoti_fattn_block_mass_B = 0; ++ opencoti_fattn_block_mass_qwin = 0; ++ GGML_UNUSED(block_mass); GGML_UNUSED(block_mass_nblock); ++ GGML_UNUSED(block_mass_B); GGML_UNUSED(block_mass_qwin); ++ + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; +@@ -1195,6 +1222,11 @@ void launch_fattn( + + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; ++ // opencoti-hook: sparse-attn #551 — block_sel rides src[7] (src[5]/src[6] are the streaming/ ++ // position-window k_tail/v_tail). Its op_params[0]=block_size; ne[0]=n_block (the −1-padded list stride). ++ const ggml_tensor * block_sel = dst->src[7]; ++ const int sparse_B = block_sel ? ((const int32_t *) block_sel->op_params)[0] : 0; ++ const int sparse_nwords = block_sel ? (int) block_sel->ne[0] : 0; + + ggml_tensor * KQV = dst; + +@@ -1398,6 +1430,9 @@ void launch_fattn( + } + } + ++ // opencoti #551 bug-743: parallel_blocks=1 discriminator (2026-06-26) PROVED the gridDim.y combine ++ // is NOT the cause — sparse collapsed to niah=0 even with combine removed. Reverted; the bug is in ++ // the produce side (select cnt) or per-tile walk, now under select-kernel instrumentation. + blocks_num.x = ntiles_x; + blocks_num.y = parallel_blocks; + blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3]; +@@ -1440,6 +1475,8 @@ void launch_fattn( + sinks ? ((const char *) sinks->data) : nullptr, + KV_max.ptr, + KV_min.ptr, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band (nullptr when prepass didn't run) ++ block_sel ? (const int *) block_sel->data : nullptr, // opencoti-hook: sparse-attn #551 ++ sparse_B, sparse_nwords, // opencoti-hook: sparse-attn #551 + !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, logit_softcap, + Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +index d12da11..185eaef 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -1936,6 +1936,9 @@ static __global__ void flash_attn_ext_f16( + const char * __restrict__ sinks, + const int * __restrict__ KV_max, + const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim ++ const int * __restrict__ block_sel, // opencoti-hook: sparse-attn #551 — unused in MMA (decode uses VEC) ++ const int32_t sparse_B, // opencoti-hook: sparse-attn #551 ++ const int32_t sparse_nwords, // opencoti-hook: sparse-attn #551 + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -2103,7 +2106,7 @@ static __global__ void flash_attn_ext_f16( + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh +index c0f030b..533d5c6 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh +@@ -795,6 +795,9 @@ static __global__ void flash_attn_tile( + const char * __restrict__ sinks, + const int * __restrict__ KV_max, + const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim ++ const int * __restrict__ block_sel, // opencoti-hook: sparse-attn #551 — unused in tile (decode uses VEC) ++ const int32_t sparse_B, // opencoti-hook: sparse-attn #551 ++ const int32_t sparse_nwords, // opencoti-hook: sparse-attn #551 + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -820,7 +823,7 @@ static __global__ void flash_attn_tile( + #endif // GGML_USE_WMMA_FATTN + (use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512)) + ) { +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -1135,7 +1138,7 @@ static __global__ void flash_attn_tile( + } + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh +index a4ab82b..0a2615e 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-turbo.cuh +@@ -28,18 +28,85 @@ static __constant__ float FATTN_TURBO2_CENTROIDS[4] = { + -0.133462f, -0.039994f, 0.039994f, 0.133462f }; + + // ---- per-element rotated-value decode (centroid[code]×norm). p = position within the 128-block. ---- +-static __device__ __forceinline__ float fattn_turbo3_val(const block_turbo3_0 * blk, int p, float norm) { ++// opencoti-hook: #535b turbo decode smem-codebook — `cb` is the centroid table; callers pass either the ++// __constant__ FATTN_TURBO*_CENTROIDS symbol (legacy/func-ptr path) or a per-block smem copy (the fast ++// path, mirroring the #534 TCQ treatment) to avoid __constant__ serializing the data-dependent per-element ++// lookups up to 32-way per warp. Byte-identical either way (same values/index). ++static __device__ __forceinline__ float fattn_turbo3_val(const block_turbo3_0 * blk, int p, float norm, const float * __restrict__ cb) { + const uint8_t lo = (blk->qs[p >> 2] >> ((p & 3) * 2)) & 0x3; + const uint8_t hi = (blk->signs[p >> 3] >> (p & 7)) & 0x1; +- return FATTN_TURBO3_CENTROIDS[lo | (hi << 2)] * norm; ++ return cb[lo | (hi << 2)] * norm; + } +-static __device__ __forceinline__ float fattn_turbo4_val(const block_turbo4_0 * blk, int p, float norm) { ++static __device__ __forceinline__ float fattn_turbo4_val(const block_turbo4_0 * blk, int p, float norm, const float * __restrict__ cb) { + const uint8_t idx = (blk->qs[p >> 1] >> ((p & 1) * 4)) & 0xF; +- return FATTN_TURBO4_CENTROIDS[idx] * norm; ++ return cb[idx] * norm; + } +-static __device__ __forceinline__ float fattn_turbo2_val(const block_turbo2_0 * blk, int p, float norm) { ++static __device__ __forceinline__ float fattn_turbo2_val(const block_turbo2_0 * blk, int p, float norm, const float * __restrict__ cb) { + const uint8_t idx = (blk->qs[p >> 2] >> ((p & 3) * 2)) & 0x3; +- return FATTN_TURBO2_CENTROIDS[idx] * norm; ++ return cb[idx] * norm; ++} ++ ++// opencoti-hook: fattn-vec turbo K-read vectorized loads (#535 — see docs/evaluations/context.md). ++// The KQ body is global-load-latency-bound (council csl-...0805-230a): the SM stalls ~400cyc on the ++// per-element qs/signs byte loads, not on compute (ncu 4.5%/4.5%) nor the __constant__ LUT. Each lane ++// owns ONE block-aligned, NON-straddling group of 2*NE2 (=8) consecutive elements per outer iter, so we ++// can load the whole group's qs/signs as ONE wide aligned word + hoist norm ONCE, instead of ~8 scattered ++// byte loads + 4 norm reloads. BYTE-IDENTICAL to the fattn_turboN_val path (same bits → same centroid×norm ++// → same MAD order); only the load width changes. ib is 2*NE2-aligned ⇒ qs+(ib>>2) is uint16-aligned ++// (turbo2/3) and qs+(ib>>1) is uint32-aligned (turbo4); blk base nb (14/10/68) preserves it. NE2!=4 keeps ++// the scalar fallback (non-sm_86 cpy widths). norm is the hoisted __half2float(blk->norm) for this block. ++template ++static __device__ __forceinline__ void fattn_turbo3_decode_group( ++ const block_turbo3_0 * __restrict__ blk, int ib, float norm, half2 * __restrict__ tmp, const float * __restrict__ cb) { ++ if constexpr (NE2 == 4) { ++ const uint16_t qs2 = *(const uint16_t *)(blk->qs + (ib >> 2)); // 8 × 2-bit (low), aligned ++ const uint8_t s1 = blk->signs[ib >> 3]; // 8 × 1-bit (sign) ++ #pragma unroll ++ for (int k1 = 0; k1 < 4; ++k1) { ++ const int j0 = 2*k1, j1 = j0 + 1; ++ tmp[k1] = make_half2( ++ cb[((qs2 >> (j0*2)) & 0x3) | (((s1 >> j0) & 0x1) << 2)] * norm, ++ cb[((qs2 >> (j1*2)) & 0x3) | (((s1 >> j1) & 0x1) << 2)] * norm); ++ } ++ } else { ++ #pragma unroll ++ for (int k1 = 0; k1 < NE2; ++k1) ++ tmp[k1] = make_half2(fattn_turbo3_val(blk, ib + 2*k1, norm, cb), fattn_turbo3_val(blk, ib + 2*k1 + 1, norm, cb)); ++ } ++} ++template ++static __device__ __forceinline__ void fattn_turbo4_decode_group( ++ const block_turbo4_0 * __restrict__ blk, int ib, float norm, half2 * __restrict__ tmp, const float * __restrict__ cb) { ++ if constexpr (NE2 == 4) { ++ const uint32_t qs4 = *(const uint32_t *)(blk->qs + (ib >> 1)); // 8 × 4-bit nibbles, aligned ++ #pragma unroll ++ for (int k1 = 0; k1 < 4; ++k1) { ++ const int j0 = 2*k1, j1 = j0 + 1; ++ tmp[k1] = make_half2(cb[(qs4 >> (j0*4)) & 0xF] * norm, ++ cb[(qs4 >> (j1*4)) & 0xF] * norm); ++ } ++ } else { ++ #pragma unroll ++ for (int k1 = 0; k1 < NE2; ++k1) ++ tmp[k1] = make_half2(fattn_turbo4_val(blk, ib + 2*k1, norm, cb), fattn_turbo4_val(blk, ib + 2*k1 + 1, norm, cb)); ++ } ++} ++template ++static __device__ __forceinline__ void fattn_turbo2_decode_group( ++ const block_turbo2_0 * __restrict__ blk, int ib, float norm, half2 * __restrict__ tmp, const float * __restrict__ cb) { ++ if constexpr (NE2 == 4) { ++ const uint16_t qs2 = *(const uint16_t *)(blk->qs + (ib >> 2)); // 8 × 2-bit, aligned ++ #pragma unroll ++ for (int k1 = 0; k1 < 4; ++k1) { ++ const int j0 = 2*k1, j1 = j0 + 1; ++ tmp[k1] = make_half2(cb[(qs2 >> (j0*2)) & 0x3] * norm, ++ cb[(qs2 >> (j1*2)) & 0x3] * norm); ++ } ++ } else { ++ #pragma unroll ++ for (int k1 = 0; k1 < NE2; ++k1) ++ tmp[k1] = make_half2(fattn_turbo2_val(blk, ib + 2*k1, norm, cb), fattn_turbo2_val(blk, ib + 2*k1 + 1, norm, cb)); ++ } + } + + // Q·K MAD against the float Q slice — Q_reg is half2 with V_DOT2_F32_F16_AVAILABLE, else float2 +@@ -55,23 +122,19 @@ static __device__ __forceinline__ void fattn_turbo_qmad(float & sum, half2 kval, + // ===================== K: vec_dot (centroid×norm · float Q), f16-structured ===================== + // K_c → key row in turbo format; Q_v → this lane's float Q slice (Q_reg, already ×scale); the dot + // covers this thread's D-slice, warp_reduce_sum combines lanes (done by the caller). +-#define FATTN_TURBO_KQ_BODY(BLOCK_T, VAL_FN) \ ++#define FATTN_TURBO_KQ_BODY(BLOCK_T, DECODE_GROUP, CB) \ + GGML_UNUSED(Q_q8); GGML_UNUSED(Q_ds_v); \ + constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes(); \ + constexpr int cpy_ne = cpy_nb / 4; \ + float sum = 0.0f; \ + _Pragma("unroll") \ + for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) { \ ++ /* this lane's group = 2*cpy_ne consecutive, block-aligned (no-straddle) elements */ \ ++ const int e_base = 2*(k_KQ_0 + (int)(threadIdx.x % nthreads)*cpy_ne); \ ++ const BLOCK_T * blk = (const BLOCK_T *) K_c + (e_base >> 7); \ ++ const float norm = __half2float(blk->norm); /* hoisted: once per group */ \ + half2 tmp[cpy_ne]; \ +- _Pragma("unroll") \ +- for (int k1 = 0; k1 < cpy_ne; ++k1) { \ +- const int e2 = k_KQ_0 + (int)(threadIdx.x % nthreads)*cpy_ne + k1; \ +- const int e = 2*e2; \ +- const BLOCK_T * blk = (const BLOCK_T *) K_c + (e >> 7); \ +- const float norm = __half2float(blk->norm); \ +- const int ib = e & 127; \ +- tmp[k1] = make_half2(VAL_FN(blk, ib, norm), VAL_FN(blk, ib + 1, norm)); \ +- } \ ++ DECODE_GROUP(blk, e_base & 127, norm, tmp, (CB)); /* one wide qs/signs load */ \ + _Pragma("unroll") \ + for (int k1 = 0; k1 < cpy_ne; ++k1) { \ + fattn_turbo_qmad(sum, tmp[k1], Q_v, k_KQ_0/nthreads + k1); \ +@@ -79,32 +142,61 @@ static __device__ __forceinline__ void fattn_turbo_qmad(float & sum, half2 kval, + } \ + return sum; + ++// Legacy __constant__-codebook K readers (kept for the function-pointer dispatch typedef; behaviour ++// unchanged — they pass the __constant__ symbol as the codebook). + template + static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { +- FATTN_TURBO_KQ_BODY(block_turbo3_0, fattn_turbo3_val) ++ FATTN_TURBO_KQ_BODY(block_turbo3_0, fattn_turbo3_decode_group, FATTN_TURBO3_CENTROIDS) + } + template + static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo4( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { +- FATTN_TURBO_KQ_BODY(block_turbo4_0, fattn_turbo4_val) ++ FATTN_TURBO_KQ_BODY(block_turbo4_0, fattn_turbo4_decode_group, FATTN_TURBO4_CENTROIDS) + } + template + static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2( + const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) { +- FATTN_TURBO_KQ_BODY(block_turbo2_0, fattn_turbo2_val) ++ FATTN_TURBO_KQ_BODY(block_turbo2_0, fattn_turbo2_decode_group, FATTN_TURBO2_CENTROIDS) ++} ++ ++// opencoti-hook: #535b — smem-codebook K readers (the fast path). `cb` is the per-block smem centroid ++// copy staged once at the kernel head; avoids the __constant__ per-warp serialization. Byte-identical. ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3_smem( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, const float * __restrict__ cb) { ++ FATTN_TURBO_KQ_BODY(block_turbo3_0, fattn_turbo3_decode_group, cb) ++} ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo4_smem( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, const float * __restrict__ cb) { ++ FATTN_TURBO_KQ_BODY(block_turbo4_0, fattn_turbo4_decode_group, cb) ++} ++template ++static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2_smem( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, const float * __restrict__ cb) { ++ FATTN_TURBO_KQ_BODY(block_turbo2_0, fattn_turbo2_decode_group, cb) ++} ++ ++// Compile-time tier dispatch for the smem K read (mirrors tcq_vec_dot_KQ_smem in fattn-vec.cuh). ++template ++static __device__ __forceinline__ float turbo_vec_dot_KQ_smem( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, const float * __restrict__ cb) { ++ if constexpr (type_K == GGML_TYPE_TURBO3_0) return vec_dot_fattn_vec_KQ_turbo3_smem(K_c, Q_v, Q_q8, Q_ds_v, cb); ++ else if constexpr (type_K == GGML_TYPE_TURBO4_0) return vec_dot_fattn_vec_KQ_turbo4_smem(K_c, Q_v, Q_q8, Q_ds_v, cb); ++ else return vec_dot_fattn_vec_KQ_turbo2_smem(K_c, Q_v, Q_q8, Q_ds_v, cb); + } + + // ===================== V: dequant (centroid×norm), ne consecutive (i0 4-aligned) ===================== + // Reads ne consecutive rotated V values at element offset i0; writes T (half/float) as half2/float2. +-#define FATTN_TURBO_V_BODY(BLOCK_T, VAL_FN) \ ++#define FATTN_TURBO_V_BODY(BLOCK_T, VAL_FN, CB) \ + static_assert(ne % 2 == 0, "bad ne"); \ + const BLOCK_T * blk = (const BLOCK_T *) vx + (i0 >> 7); \ + const float norm = __half2float(blk->norm); \ + const int ib = (int)(i0 & 127); \ + float vals[ne]; \ + _Pragma("unroll") \ +- for (int l = 0; l < ne; ++l) vals[l] = VAL_FN(blk, ib + l, norm); \ ++ for (int l = 0; l < ne; ++l) vals[l] = VAL_FN(blk, ib + l, norm, (CB)); \ + if constexpr (std::is_same_v) { \ + half2 * d2 = (half2 *) dst; \ + _Pragma("unroll") \ +@@ -115,15 +207,38 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2( + for (int l = 0; l < ne; l += 2) d2[l/2] = make_float2(vals[l], vals[l+1]); \ + } + ++// Legacy __constant__-codebook V dequant (function-pointer typedef path; behaviour unchanged). + template + static __device__ __forceinline__ void dequantize_V_turbo3(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { +- FATTN_TURBO_V_BODY(block_turbo3_0, fattn_turbo3_val) ++ FATTN_TURBO_V_BODY(block_turbo3_0, fattn_turbo3_val, FATTN_TURBO3_CENTROIDS) + } + template + static __device__ __forceinline__ void dequantize_V_turbo4(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { +- FATTN_TURBO_V_BODY(block_turbo4_0, fattn_turbo4_val) ++ FATTN_TURBO_V_BODY(block_turbo4_0, fattn_turbo4_val, FATTN_TURBO4_CENTROIDS) + } + template + static __device__ __forceinline__ void dequantize_V_turbo2(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { +- FATTN_TURBO_V_BODY(block_turbo2_0, fattn_turbo2_val) ++ FATTN_TURBO_V_BODY(block_turbo2_0, fattn_turbo2_val, FATTN_TURBO2_CENTROIDS) ++} ++ ++// opencoti-hook: #535b — smem-codebook V dequant (fast path). `cb` = per-block smem centroid copy. ++template ++static __device__ __forceinline__ void dequantize_V_turbo3_smem(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, const float * __restrict__ cb) { ++ FATTN_TURBO_V_BODY(block_turbo3_0, fattn_turbo3_val, cb) ++} ++template ++static __device__ __forceinline__ void dequantize_V_turbo4_smem(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, const float * __restrict__ cb) { ++ FATTN_TURBO_V_BODY(block_turbo4_0, fattn_turbo4_val, cb) ++} ++template ++static __device__ __forceinline__ void dequantize_V_turbo2_smem(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, const float * __restrict__ cb) { ++ FATTN_TURBO_V_BODY(block_turbo2_0, fattn_turbo2_val, cb) ++} ++ ++// Compile-time tier dispatch for the smem V dequant (mirrors tcq_dequantize_V_smem in fattn-vec.cuh). ++template ++static __device__ __forceinline__ void turbo_dequantize_V_smem(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, const float * __restrict__ cb) { ++ if constexpr (type_V == GGML_TYPE_TURBO3_0) dequantize_V_turbo3_smem(vx, dst, i0, cb); ++ else if constexpr (type_V == GGML_TYPE_TURBO4_0) dequantize_V_turbo4_smem(vx, dst, i0, cb); ++ else dequantize_V_turbo2_smem(vx, dst, i0, cb); + } +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +index abcce88..949db5e 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -1,5 +1,24 @@ + #include "common.cuh" + #include "fattn-common.cuh" ++#include // opencoti: getenv/atof for sparse-v (#546) ++ ++// opencoti-hook: sparse-v (#546) — attention-gated V-dequant skip (theTom/turboquant_plus port). ++// Per-compilation-unit copy (no device RDC); broadcast from the host in case_impl below. Default 0.0 ++// ⇒ the skip predicate (weight < tau) is never true ⇒ byte-identical to the dense V-read. ++static __constant__ float d_sparse_v_tau = 0.0f; ++// opencoti-hook: sparse-v adaptive-τ (#565) — mass-budget knob. When >0 it OVERRIDES the static τ with a ++// runtime, distribution-aware threshold θ_j = (eps/n_kv)·KQ_sum[j] (eps × mean post-exp weight per col): ++// skip key k iff its normalized contribution exp(S−m)/L < eps/n_kv for EVERY query column ⇒ total dropped ++// softmax mass < eps by construction, and θ auto-tracks ctx length + attention diffuseness (the long-ctx ++// failure of a fixed peak-relative τ). eps==0 ⇒ static-τ path, byte-identical. ++static __constant__ float d_sparse_v_eps = 0.0f; ++// opencoti-hook: sparse-v combined (#565) — sink + recent ALWAYS-KEEP guards. The eps/tau skip is a ++// peak/mean-referenced predicate that (a) under attention-sink wrongly drops mid-ctx peaks, and (b) drops ++// recent tokens that MTP draft/verify depend on. Always keep pos=n_kv-recent ++// (recent window incl. spec-draft tokens) so the skip is lossless on the needle AND preserves draft acceptance. ++// Both default 0 ⇒ no protection (byte-identical to pre-#565 when eps==tau==0). ++static __constant__ int d_sparse_v_sink = 0; ++static __constant__ int d_sparse_v_recent = 0; + + static int ggml_cuda_fattn_vec_get_nthreads_host(const int cc) { + return 128; +@@ -10,14 +29,64 @@ static constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() { + return 128; + } + ++// opencoti-hook: fattn-vec occupancy lift (#533/#535) — see docs/evaluations/context.md §FA-VEC-occupancy. ++// The decode FA-VEC kernel was register-bound: nvcc grabbed 255 regs/thread (the hardware ceiling — it was ++// already SPILLING to local memory, STACK:80) because __launch_bounds__'s min-blocks-per-SM was 1. That caps ++// the kernel at 2 blocks/SM → 16.7% theoretical / 8.3% achieved occupancy on sm_86 (48 warps/SM max), so the ++// GPU sat ~90% idle and the kernel ran LATENCY-bound (ncu: compute 4.5%, mem 4.5% — neither saturated). With ++// only ~4 active warps there was nothing to hide the per-key K/V read latency. Forcing a higher min-blocks ++// makes nvcc cap registers (65536/(128*N)) so N blocks co-reside, raising occupancy to hide that latency. ++// Helps ALL KV types, and ESPECIALLY turbo/TCQ whose centroid/codebook unpack has the longest dep-chain (the ++// most latency to hide). NUMERICS ARE BYTE-IDENTICAL — this only changes block scheduling, not the math. ++// Tune here (sm_86, 128 thr): 2→≤256regs/16.7%occ · 3→≤170regs/25% · 4→≤128regs/33% · 6→≤85regs/50%. ++#ifndef GGML_CUDA_FATTN_VEC_MIN_BLOCKS_PER_SM ++#define GGML_CUDA_FATTN_VEC_MIN_BLOCKS_PER_SM 3 ++#endif ++// min-blocks is D-dependent (measured ptxas register/spill curve, sm_86, turbo3 D-rungs): ++// D<=256 (A4B local layers, Qwen D128, the COMMON decode path): min-blocks=3 → nvcc caps at 168 regs ++// and finds a CLEAN allocation — ~0 spill (8B stack / 4B spill vs 64B/72B at min-blocks=1) AND ++// occupancy 16.7%→25%. A genuine free win: more warps to hide read latency, less local-mem traffic. ++// D==512 (Gemma-global / 27B head_dim): already register-thrashing at the old budget (240B spill at ++// 255 regs — the root of its pathological decode cost); a tighter budget only worsens it ++// (mb=3 → 756B spill / 2236B loads). No clean launch-bounds point exists for D512; it needs a ++// structural register cut (turbo-read restructure), tracked separately. So D512 keeps min-blocks=2. ++static constexpr __device__ int ggml_cuda_fattn_vec_min_blocks(int D) { ++ // D<=256 → 3 blocks/SM (168 regs, ~0 spill, 25% occupancy vs the old 16.7%). D512 keeps the upstream ++ // min_blocks=2: at 168 regs it would spill 240B (the D-aware nthreads_KQ that avoided this regressed ++ // the kernel, see above), and decode tps was flat regardless of D512 occupancy. NOTE (#535): this ++ // occupancy lift is byte-identical but only marginally moved tps (decode is FFN/MoE-bound + the turbo ++ // gap is the read dep-chain, not occupancy); kept because it's free, not because it closed the gap. ++ return D <= 256 ? GGML_CUDA_FATTN_VEC_MIN_BLOCKS_PER_SM : 2; ++} ++ + // Currently llvm with the amdgcn target does not support unrolling loops + // that contain a break that can not be resolved at compile time. + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wpass-failed" + #endif // __clang__ ++// opencoti-hook: #533 TCQ decode smem-codebook dispatch — the FA-VEC TCQ readers normally index a ++// 512/256-entry codebook in __constant__ memory; with data-dependent (per-element) indices that ++// serializes up to 32-way per warp. These thin helpers call the _cb reader variants (fattn-tcq.cuh, ++// which take an explicit codebook pointer) so the kernel can point them at a shared-memory copy. Picks ++// the tier at compile time; only instantiated under `if constexpr (is_tcq_*)`, so non-TCQ kernels never ++// see them. See docs/features/poly_kv.md / UPSTREAM_SYNC.md. ++template ++static __device__ __forceinline__ float tcq_vec_dot_KQ_smem( ++ const char * __restrict__ K_c, const void * __restrict__ Q_v, ++ const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v, const float * __restrict__ cb) { ++ if constexpr (tK == GGML_TYPE_TURBO3_TCQ) return vec_dot_fattn_vec_KQ_turbo3_tcq_cb(K_c, Q_v, Q_q8, Q_ds_v, cb); ++ else return vec_dot_fattn_vec_KQ_turbo2_tcq_cb(K_c, Q_v, Q_q8, Q_ds_v, cb); ++} ++template ++static __device__ __forceinline__ void tcq_dequantize_V_smem( ++ const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0, const float * __restrict__ cb) { ++ if constexpr (tV == GGML_TYPE_TURBO3_TCQ) dequantize_V_turbo3_tcq_cb(vx, dst, i0, cb); ++ else dequantize_V_turbo2_tcq_cb(vx, dst, i0, cb); ++} ++ + template // D == head size +-__launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1) ++__launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), ggml_cuda_fattn_vec_min_blocks(D)) + static __global__ void flash_attn_ext_vec( + const char * __restrict__ Q, + const char * __restrict__ K, +@@ -26,6 +95,9 @@ static __global__ void flash_attn_ext_vec( + const char * __restrict__ sinks, + const int * __restrict__ KV_max, + const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim ++ const int * __restrict__ block_sel, // opencoti-hook: sparse-attn #551 — sorted −1-padded selected-block list (nullptr=dense) ++ const int32_t sparse_B, // opencoti-hook: sparse-attn #551 — block size (multiple of nthreads) ++ const int32_t sparse_nwords, // opencoti-hook: sparse-attn #551 — list stride == n_block per sequence + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -46,7 +118,7 @@ static __global__ void flash_attn_ext_vec( + + // Skip unused kernel variants for faster compilation: + if (use_logit_softcap && !(D == 128 || D == 256)) { +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -85,6 +157,10 @@ static __global__ void flash_attn_ext_vec( + // WHT-rotation precision the trellis reconstruction needs. V uses our quantized split (ne=4), same as turbo_0. + || type_K == GGML_TYPE_TURBO3_TCQ || type_K == GGML_TYPE_TURBO2_TCQ); + constexpr int nthreads = ggml_cuda_fattn_vec_get_nthreads_device(); ++ // opencoti #535 NOTE: tried D-aware nthreads_KQ for turbo D512 (32 lanes → Q_reg 64→16, 168reg/0spill), ++ // but it REGRESSED the D512 decode kernel 287µs→338µs (+18%): the benign L1-cached spill was cheaper ++ // than the 32-lane warp reduction it introduced, and decode tps was flat regardless (D512 attn is a ++ // small decode slice). Reverted to the f16-style fixed split. See docs/evaluations/context.md. + constexpr int nthreads_KQ = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16 || is_turbo_K) ? 128 / cpy_nb : nthreads_KQ_q; + constexpr int nthreads_V = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_V_q; + +@@ -250,20 +326,90 @@ static __global__ void flash_attn_ext_vec( + #endif // V_DOT2_F32_F16_AVAILABLE + } + ++ // opencoti-hook: #533 TCQ decode smem-codebook — copy the per-tier codebook into shared memory once ++ // (cooperative over all nthreads), so the K-dot / V-dequant below read it from smem (broadcast, no ++ // bank conflict on a hit) instead of __constant__ (which serializes the data-dependent per-element ++ // lookups up to 32-way per warp). Also lets the TURBO_TCQ_CB runtime override reach decode (it writes ++ // the __constant__ symbol we copy from). ≤2 KB smem + a single block barrier at the kernel head. ++ constexpr bool is_tcq_K = (type_K == GGML_TYPE_TURBO3_TCQ || type_K == GGML_TYPE_TURBO2_TCQ); ++ constexpr bool is_tcq_V = (type_V == GGML_TYPE_TURBO3_TCQ || type_V == GGML_TYPE_TURBO2_TCQ); ++ constexpr int cb_K_n = type_K == GGML_TYPE_TURBO3_TCQ ? 512 : (type_K == GGML_TYPE_TURBO2_TCQ ? 256 : 1); ++ constexpr int cb_V_n = type_V == GGML_TYPE_TURBO3_TCQ ? 512 : (type_V == GGML_TYPE_TURBO2_TCQ ? 256 : 1); ++ __shared__ float tcq_cb_K[cb_K_n]; ++ __shared__ float tcq_cb_V[cb_V_n]; ++ if constexpr (is_tcq_K) { ++ const float * src_K = (type_K == GGML_TYPE_TURBO3_TCQ) ? d_turbo3_tcq_codebook_fattn : d_turbo2_tcq_codebook_fattn; ++ for (int i = tid; i < cb_K_n; i += nthreads) tcq_cb_K[i] = src_K[i]; ++ } ++ if constexpr (is_tcq_V) { ++ const float * src_V = (type_V == GGML_TYPE_TURBO3_TCQ) ? d_turbo3_tcq_codebook_fattn : d_turbo2_tcq_codebook_fattn; ++ for (int i = tid; i < cb_V_n; i += nthreads) tcq_cb_V[i] = src_V[i]; ++ } ++ // opencoti-hook: #535b plain-turbo decode smem-codebook — same treatment as the TCQ tiers above, for ++ // the non-trellis turbo2/3/4_0 centroid tables (4/8/16 entries). __constant__ serializes the ++ // data-dependent per-element centroid lookups up to 32-way/warp; smem broadcasts. This is the proven ++ // root of the turbo3_tcq(49µs) < turbo3(63µs) D256 K-read gap (#534 fixed it for TCQ only). Byte-identical. ++ constexpr bool is_turbo0_K = (type_K == GGML_TYPE_TURBO2_0 || type_K == GGML_TYPE_TURBO3_0 || type_K == GGML_TYPE_TURBO4_0); ++ constexpr bool is_turbo0_V = (type_V == GGML_TYPE_TURBO2_0 || type_V == GGML_TYPE_TURBO3_0 || type_V == GGML_TYPE_TURBO4_0); ++ constexpr int cb_t0_K_n = type_K == GGML_TYPE_TURBO3_0 ? 8 : (type_K == GGML_TYPE_TURBO4_0 ? 16 : (type_K == GGML_TYPE_TURBO2_0 ? 4 : 1)); ++ constexpr int cb_t0_V_n = type_V == GGML_TYPE_TURBO3_0 ? 8 : (type_V == GGML_TYPE_TURBO4_0 ? 16 : (type_V == GGML_TYPE_TURBO2_0 ? 4 : 1)); ++ __shared__ float turbo_cb_K[cb_t0_K_n]; ++ __shared__ float turbo_cb_V[cb_t0_V_n]; ++ if constexpr (is_turbo0_K) { ++ const float * src_K = (type_K == GGML_TYPE_TURBO3_0) ? FATTN_TURBO3_CENTROIDS : (type_K == GGML_TYPE_TURBO4_0 ? FATTN_TURBO4_CENTROIDS : FATTN_TURBO2_CENTROIDS); ++ for (int i = tid; i < cb_t0_K_n; i += nthreads) turbo_cb_K[i] = src_K[i]; ++ } ++ if constexpr (is_turbo0_V) { ++ const float * src_V = (type_V == GGML_TYPE_TURBO3_0) ? FATTN_TURBO3_CENTROIDS : (type_V == GGML_TYPE_TURBO4_0 ? FATTN_TURBO4_CENTROIDS : FATTN_TURBO2_CENTROIDS); ++ for (int i = tid; i < cb_t0_V_n; i += nthreads) turbo_cb_V[i] = src_V[i]; ++ } ++ if constexpr (is_tcq_K || is_tcq_V || is_turbo0_K || is_turbo0_V) { __syncthreads(); } ++ + const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; + const int k_VKQ_min = KV_min ? KV_min[sequence*gridDim.x + blockIdx.x] : 0; // opencoti F5 dca perf (T87.pD-opt O4) +- K += blockIdx.y*nthreads * nb11; +- V += blockIdx.y*nthreads * nb21; +- maskh += blockIdx.y*nthreads; +- for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads, +- // Increment pointers after each loop: +- K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) { ++ // opencoti-hook: sparse-attn #551 — compacted selected-block iteration (council Rank-1, csl-2026-06-26-0516). ++ // Decode is weight-bound; the old "iterate every tile, continue on unselected" loop paid the per-tile branch + ++ // K/V/mask pointer-advance on the ~3/4 skipped tiles with no compensating saving (−6% vs dense). Here the ++ // selector emits a SORTED, −1-padded list of selected block indices (sparse_nwords == n_block entries/seq); we ++ // iterate ONLY those, jumping K/V/mask directly to each tile — O(selected), not O(capacity). The cumulative ++ // offset of the original strided loop at tile k_VKQ_0 is exactly base + k_VKQ_0 (mask elements / K,V bytes via ++ // nb11,nb21), so a from-base recompute is identical. Dense (block_sel==nullptr) walks every tile in that same ++ // strided order ⇒ byte-identical; topk=0 ⇒ list is [0,1,2,…] ⇒ also byte-identical. A gridDim.y block assigned ++ // zero tiles falls through to the identity partial write (KQ_max=−FLT_MAX/2, KQ_sum=0) below, leaving the ++ // online-softmax combine unaffected. sparse_B is a multiple of nthreads (enforced graph-side), so sparse_m≥1. ++ const char * const K_base = K; ++ const char * const V_base = V; ++ const auto maskh_base = maskh; ++ const int * const sparse_sel = block_sel ? block_sel + sequence*sparse_nwords : nullptr; // sorted, −1-padded ++ const int sparse_m = block_sel ? (sparse_B / nthreads) : 1; // tiles per selected block (≥1) ++ const int sparse_nw = block_sel ? sparse_nwords*sparse_m // upper bound: n_block*m tiles ++ : (k_VKQ_max + nthreads - 1)/nthreads; ++ for (int kv_it = blockIdx.y; kv_it < sparse_nw; kv_it += gridDim.y) { ++ int k_VKQ_0; ++ if (sparse_sel) { ++ const int blk = sparse_sel[kv_it / sparse_m]; ++ if (blk < 0) { // −1 sentinel: past the last selected block ++ break; ++ } ++ k_VKQ_0 = blk*sparse_B + (kv_it % sparse_m)*nthreads; ++ if (k_VKQ_0 >= k_VKQ_max) { // selected block beyond valid KV / DCA high edge ++ continue; ++ } ++ } else { ++ k_VKQ_0 = kv_it * nthreads; ++ if (k_VKQ_0 >= k_VKQ_max) { ++ break; ++ } ++ } + // opencoti F5 dca perf (T87.pD-opt O4): skip K/V tiles entirely below the low band edge +- // (DCA INTRA/SUCC regimes, SWA windows). k_VKQ_min==0 for causal masks → never skips. The +- // for-increment above still advances the K/V/mask pointers, so they stay in sync. ++ // (DCA INTRA/SUCC regimes, SWA windows). k_VKQ_min==0 for causal masks → never skips. + if (k_VKQ_0 + nthreads <= k_VKQ_min) { + continue; + } ++ // Jump K/V/mask directly to this tile (char* byte strides nb11/nb21; maskh element stride). ++ K = K_base + (size_t) k_VKQ_0 * nb11; ++ V = V_base + (size_t) k_VKQ_0 * nb21; ++ maskh = maskh_base + k_VKQ_0; + + // Calculate KQ tile and keep track of new maximum KQ values: + float KQ_reg[ncols]; // KQ in registers. +@@ -280,7 +426,14 @@ static __global__ void flash_attn_ext_vec( + + #pragma unroll + for (int j = 0; j < ncols; ++j) { +- float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]); ++ float sum; ++ if constexpr (is_tcq_K) { // opencoti #533: read codebook from smem, not __constant__ ++ sum = tcq_vec_dot_KQ_smem(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j], tcq_cb_K); ++ } else if constexpr (is_turbo0_K) { // opencoti #535b: plain-turbo smem codebook ++ sum = turbo_vec_dot_KQ_smem(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j], turbo_cb_K); ++ } else { ++ sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]); ++ } + sum = warp_reduce_sum(sum); + + if (use_logit_softcap) { +@@ -341,6 +494,30 @@ static __global__ void flash_attn_ext_vec( + for (int j = 0; j < ncols; ++j) { + KQ_k[j] = __half2half2(KQ[j*nthreads + k]); + } ++ // opencoti-hook: sparse-v (#546) — skip V-dequant+accumulate for negligible-weight positions. ++ // Quantized-V only (f16/bf16 dequant is trivial). Divergent per-thread continue is safe here: no ++ // smem write / __syncwarp in this k0 iteration, and a skipped position's weighted V ≈ 0 (its tiny ++ // weight stays in the softmax normalizer, so the error is bounded by tau ⇒ below f16 noise). ++ if constexpr (type_V != GGML_TYPE_F16 && type_V != GGML_TYPE_BF16) { ++ // opencoti-hook: sparse-v combined skip (#546/#565). pos = absolute KV index of this key. ++ const int pos = k_VKQ_0 + k; ++ const int n_kv = (int) ne11; ++ if (pos >= d_sparse_v_sink && pos < n_kv - d_sparse_v_recent) { ++ // #565 partial-L FIX: threshold = eps × RUNNING MEAN (KQ_sum / keys-processed), NOT ++ // eps/n_kv·KQ_sum (partial-sum ÷ total ⇒ too small early ⇒ skip vanishes as ctx grows ⇒ ++ // the observed inversion). keys summed into KQ_sum so far = k_VKQ_0 + nthreads. eps==0 ⇒ ++ // thr=d_sparse_v_tau (∀-col 0.0f) ? d_sparse_v_eps * d_inv_nproc * KQ_sum[jj] : d_sparse_v_tau; ++ if (w >= thr) { opencoti_skip = false; break; } ++ } ++ if (opencoti_skip) { continue; } ++ } ++ } + #pragma unroll + for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) { + half2 tmp[V_rows_per_thread/2]; +@@ -352,6 +529,12 @@ static __global__ void flash_attn_ext_vec( + for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) { + tmp[i_VKQ_1] = __float22half2_rn(tmp_f[i_VKQ_1]); + } ++ } else if constexpr (is_tcq_V) { // opencoti #533: read codebook from smem, not __constant__ ++ tcq_dequantize_V_smem(V + k*nb21, tmp, ++ 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread, tcq_cb_V); ++ } else if constexpr (is_turbo0_V) { // opencoti #535b: plain-turbo smem codebook ++ turbo_dequantize_V_smem(V + k*nb21, tmp, ++ 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread, turbo_cb_V); + } else { + dequantize_V(V + k*nb21, tmp, + 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread); +@@ -370,11 +553,35 @@ static __global__ void flash_attn_ext_vec( + for (int j = 0; j < ncols; ++j) { + KQ_k[j] = KQ[j*nthreads + k]; + } ++ // opencoti-hook: sparse-v combined skip (#546/#565) — see half2 path above. eps==tau==0 byte-identical. ++ if constexpr (type_V != GGML_TYPE_F16 && type_V != GGML_TYPE_BF16) { ++ const int pos = k_VKQ_0 + k; ++ const int n_kv = (int) ne11; ++ if (pos >= d_sparse_v_sink && pos < n_kv - d_sparse_v_recent) { ++ const float d_inv_nproc = 1.0f / (float) (k_VKQ_0 + nthreads); ++ bool opencoti_skip = true; ++#pragma unroll ++ for (int jj = 0; jj < ncols; ++jj) { ++ const float w = KQ_k[jj]; ++ const float thr = (d_sparse_v_eps > 0.0f) ? d_sparse_v_eps * d_inv_nproc * KQ_sum[jj] : d_sparse_v_tau; ++ if (w >= thr) { opencoti_skip = false; break; } ++ } ++ if (opencoti_skip) { continue; } ++ } ++ } + #pragma unroll + for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) { + float2 tmp[V_rows_per_thread/2]; +- dequantize_V(V + k*nb21, tmp, +- 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread); ++ if constexpr (is_tcq_V) { // opencoti #533: read codebook from smem, not __constant__ ++ tcq_dequantize_V_smem(V + k*nb21, tmp, ++ 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread, tcq_cb_V); ++ } else if constexpr (is_turbo0_V) { // opencoti #535b: plain-turbo smem codebook ++ turbo_dequantize_V_smem(V + k*nb21, tmp, ++ 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread, turbo_cb_V); ++ } else { ++ dequantize_V(V + k*nb21, tmp, ++ 2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread); ++ } + #pragma unroll + for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) { + #pragma unroll +@@ -524,7 +731,7 @@ static __global__ void flash_attn_ext_vec( + dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]); + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +@@ -542,6 +749,26 @@ static __global__ void flash_attn_ext_vec( + + template + void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ // opencoti-hook: sparse-v (#546) — broadcast the host threshold into THIS instance TU's __constant__ ++ // copy once (no device RDC ⇒ per-TU symbol). TURBO_SPARSE_V_TAU unset/0 ⇒ dense V-read (byte-identical). ++ static bool opencoti_sparse_v_set = false; ++ if (!opencoti_sparse_v_set) { ++ const char * e = getenv("TURBO_SPARSE_V_TAU"); ++ const float tau = e ? (float) atof(e) : 0.0f; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_sparse_v_tau, &tau, sizeof(float))); ++ // opencoti-hook: sparse-v adaptive-τ (#565) — mass-budget override; unset/0 ⇒ static-τ, byte-identical. ++ const char * ee = getenv("TURBO_SPARSE_V_EPS"); ++ const float eps = ee ? (float) atof(ee) : 0.0f; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_sparse_v_eps, &eps, sizeof(float))); ++ // opencoti-hook: sparse-v combined (#565) — sink/recent always-keep guards (unset/0 ⇒ no protection). ++ const char * es = getenv("TURBO_SPARSE_V_SINK"); ++ const int sink = es ? atoi(es) : 0; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_sparse_v_sink, &sink, sizeof(int))); ++ const char * er = getenv("TURBO_SPARSE_V_RECENT"); ++ const int recent = er ? atoi(er) : 0; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_sparse_v_recent, &recent, sizeof(int))); ++ opencoti_sparse_v_set = true; ++ } + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + + const int nthreads = ggml_cuda_fattn_vec_get_nthreads_host(cc); +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +index 0f582fc..85dac93 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +@@ -31,6 +31,9 @@ static __global__ void flash_attn_ext_f16( + const char * __restrict__ sinks, + const int * __restrict__ KV_max, + const int * __restrict__ KV_min, // opencoti F5 dca perf (T87.pD-opt O4): low-edge band trim ++ const int * __restrict__ block_sel, // opencoti-hook: sparse-attn #551 — unused in WMMA (legacy path) ++ const int32_t sparse_B, // opencoti-hook: sparse-attn #551 ++ const int32_t sparse_nwords, // opencoti-hook: sparse-attn #551 + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, +@@ -501,7 +504,7 @@ static __global__ void flash_attn_ext_f16( + dst_meta[j_dst_unrolled] = dst_meta_val; + } + #else +- GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, dst, dst_meta, scale, ++ GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, KV_min, block_sel, sparse_B, sparse_nwords, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, + ne00, ne01, ne02, ne03, + nb01, nb02, nb03, +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index 23d90a3..e53ac9c 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -28,6 +28,15 @@ thread_local const int * opencoti_fattn_dca_pos_q = nullptr; + thread_local int opencoti_fattn_dca_chunk = 0; + thread_local int opencoti_fattn_dca_regime = -1; + ++// opencoti-hook: sparse-attn-vslash side-output (#551 S1) — definitions of the block-mass side channel ++// declared in fattn-common.cuh. Set by the vslash prefill graph site before the FA dispatch; drained by ++// launch_fattn. The MMA kernel emits per-block question-window mass here (S2) when non-null; until the ++// S2 kernel emit lands these are set/drained but unconsumed (inert, byte-identical). ++thread_local float * opencoti_fattn_block_mass = nullptr; ++thread_local int opencoti_fattn_block_mass_nblock = 0; ++thread_local int opencoti_fattn_block_mass_B = 0; ++thread_local int opencoti_fattn_block_mass_qwin = 0; ++ + template + static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; +@@ -574,6 +583,16 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + return BEST_FATTN_KERNEL_VEC; + } + ++ // opencoti-hook: sparse-attn #551 (bug-742) — the block-selector skip walk lives ONLY in the FA-VEC ++ // kernel (fattn-vec.cuh compacted −1-padded list). High-GQA + long-KV decode (gqa>4 && K->ne[1]>=8192, ++ // the exact 1M-serving regime) otherwise falls to the MMA path below, which has no block_sel reader and ++ // streams the FULL KV — silently dropping the skip (measured: tps bit-invariant across k_sel 32..512). ++ // When the selector attaches a block_sel list (dst->src[7] != nullptr, i.e. sparse-attn active with ++ // topk!=0), force VEC so the skip is honored. Dense/topk0 leaves src[7]==nullptr ⇒ routing unchanged. ++ if (dst->src[7] != nullptr && can_use_vector_kernel) { ++ return BEST_FATTN_KERNEL_VEC; ++ } ++ + // If Turing tensor cores are available, use them: + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +index a6ef9ee..4db22f3 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -69,6 +69,7 @@ + #include "ggml-cuda/cumsum.cuh" + #include "ggml-cuda/fill.cuh" + #include "ggml-cuda/turbo-wht.cuh" // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #391 ++#include "ggml-cuda/sparse-attn.cuh" // opencoti-hook: sparse-attn — #551 Quest block-selector + #include "ggml.h" + + #include +@@ -3047,6 +3048,21 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg + // turbo-wht.cu. + ggml_cuda_turbo_wht(ctx, dst); + break; ++ case GGML_OP_SPARSE_ATTN_FILL: ++ // opencoti-hook: sparse-attn — #551 B1. Quest per-block kmin/kmax fill from the ++ // freshly-written cache K. See sparse-attn.cu. ++ ggml_cuda_sparse_attn_fill(ctx, dst); ++ break; ++ case GGML_OP_SPARSE_ATTN_SELECT: ++ // opencoti-hook: sparse-attn — #551 B2. Quest per-(seq) block bitmask from Q·kbounds ++ // upper bound + top-K + recent/sink. See sparse-attn.cu. ++ ggml_cuda_sparse_attn_select(ctx, dst); ++ break; ++ case GGML_OP_SPARSE_ATTN_VSLASH: ++ // opencoti-hook: sparse-attn-vslash — #551 content-driven per-(seq) block list from the ++ // actual per-block max logit (last q_win queries · cache K) + top-K + recent/sink. See sparse-attn.cu. ++ ggml_cuda_sparse_attn_vslash(ctx, dst); ++ break; + case GGML_OP_OUT_PROD: + ggml_cuda_out_prod(ctx, dst); + break; +@@ -5421,7 +5437,9 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + // build_attn_mha's ggml_cast(turboN→f16) on CUDA instead of spilling to the CPU + // backend. See docs/features/poly_kv.md (M6-S2 perf-path Level A). + if ((src0_type == GGML_TYPE_TURBO2_0 || src0_type == GGML_TYPE_TURBO3_0 || +- src0_type == GGML_TYPE_TURBO4_0 || src0_type == GGML_TYPE_TURBO8_0) && ++ src0_type == GGML_TYPE_TURBO4_0 || src0_type == GGML_TYPE_TURBO8_0 || ++ // opencoti-hook #533: TCQ→f16 dequant-on-lift (prefill-MMA + DCA-compose) ++ src0_type == GGML_TYPE_TURBO3_TCQ || src0_type == GGML_TYPE_TURBO2_TCQ) && + src1_type == GGML_TYPE_F16) { + return true; + } +@@ -5687,6 +5705,41 @@ static bool GGML_CALL ggml_backend_cuda_device_supports_op(ggml_backend_dev_t de + memcpy(&group_size, (const char *) op->op_params + sizeof(int), sizeof(int)); + return group_size == 128 && a->ne[0] % 128 == 0; + } ++ case GGML_OP_SPARSE_ATTN_FILL: { ++ // opencoti-hook: sparse-attn — #551 B1 / bug-744. f16 kbounds dst [n_embd,n_block,2] + ++ // k_cur (src[1], the NEW tokens, pre-quantization f16 OR f32) + I64-or-I32 k_idxs (src[2]). ++ // cpy_k scatters via ggml_set_rows whose indices are I64, so accept both. Reading k_cur ++ // (not the quantized cache view) is what lets the Quest bound compose with EVERY cache KV ++ // type — the cache quantization happens downstream in cpy_k; the bound is the exact pre-quant ++ // envelope (Quest's upper bound stays conservative). ++ const ggml_tensor * k = op->src[1]; ++ const ggml_tensor * k_idxs = op->src[2]; ++ if (op->type != GGML_TYPE_F16 || op->ne[2] != 2) return false; ++ if (!k || (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_F32)) return false; ++ if (!k_idxs || ++ (k_idxs->type != GGML_TYPE_I64 && k_idxs->type != GGML_TYPE_I32)) return false; ++ return true; ++ } ++ case GGML_OP_SPARSE_ATTN_SELECT: { ++ // opencoti-hook: sparse-attn — #551 B2. I32 bitmask dst; f32 query (src[0]); ++ // f16 kbounds [n_embd,n_block,2] (src[1]). ++ const ggml_tensor * q = op->src[0]; ++ const ggml_tensor * kbounds = op->src[1]; ++ if (op->type != GGML_TYPE_I32) return false; ++ if (!q || q->type != GGML_TYPE_F32) return false; ++ if (!kbounds || kbounds->type != GGML_TYPE_F16 || kbounds->ne[2] != 2) return false; ++ return true; ++ } ++ case GGML_OP_SPARSE_ATTN_VSLASH: { ++ // opencoti-hook: sparse-attn-vslash — #551. I32 list dst; f32 query (src[0]); ++ // f16 cache K (src[1], the actual K, NOT a kbounds side-cache). ++ const ggml_tensor * q = op->src[0]; ++ const ggml_tensor * k_full = op->src[1]; ++ if (op->type != GGML_TYPE_I32) return false; ++ if (!q || q->type != GGML_TYPE_F32) return false; ++ if (!k_full || k_full->type != GGML_TYPE_F16) return false; ++ return true; ++ } + case GGML_OP_CROSS_ENTROPY_LOSS: + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + case GGML_OP_OPT_STEP_ADAMW: +diff --git a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +index bd4f5a0..ccedfde 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/set-rows.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +@@ -738,6 +738,10 @@ static void set_rows_cuda_turbo3_tcq(ggml_backend_cuda_context & ctx, const ggml + // opencoti-hook: asymmetric TCQ (#513) — also override the per-TU d_turbo3 copies in the + // two asymmetric instance TUs so the runtime codebook reaches asym decode (not just symmetric). + tcq3_set_decode_codebook_t3t2(cb); tcq3_set_decode_codebook_t2t3(cb); ++ // opencoti-hook: TCQ A′ — MMA-lift materialize (#553/bug-710). Also override the ++ // turbo-cpy.cu d_turbo3_tcq_cb_cpy copy so prefill/spec-verify (TCQ→f16 lift) decodes ++ // with the same runtime codebook as FA-VEC decode (else draft⇄verify KV split). ++ tcq3_set_cpy_codebook(cb); + fprintf(stderr, "TCQ encode+decode: loaded codebook from %s (device %d)\n", cb_path, ctx.device); } + else { if (f) fclose(f); fprintf(stderr, "TCQ encode: FAILED to load codebook from %s\n", cb_path); } + } +@@ -801,6 +805,8 @@ static void set_rows_cuda_turbo2_tcq(ggml_backend_cuda_context & ctx, const ggml + // opencoti-hook: asymmetric TCQ (#513) — also override the per-TU d_turbo2 copies in the + // two asymmetric instance TUs so the runtime 2-bit codebook reaches asym decode too. + tcq2_set_decode_codebook_t3t2(cb); tcq2_set_decode_codebook_t2t3(cb); ++ // opencoti-hook: TCQ A′ — MMA-lift materialize (#553/bug-710). Mirror for the 2-bit cpy copy. ++ tcq2_set_cpy_codebook(cb); + fprintf(stderr, "TCQ2 encode+decode: loaded 2-bit codebook from %s (device %d)\n", cb_path, ctx.device); } + else { if (f) fclose(f); fprintf(stderr, "TCQ2 encode: FAILED to load codebook from %s\n", cb_path); } + } +diff --git a/bk-t1/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cu b/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cu +new file mode 100644 +index 0000000..0f8a70f +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cu +@@ -0,0 +1,537 @@ ++// opencoti-hook: sparse-attn — #551 Quest block-selector, B1 (the per-block kmin/kmax fill). ++// See docs/features/sparse_attn.md (S2/S3 design). ++// ++// Reduces the freshly-written cache K into per-(KV-head·channel, block) min/max, stored in the ++// kbounds side-cache [n_embd_k, n_block, 2] ([..,0]=min, [..,1]=max). Recompute-from-cache-K ++// (not incremental-merge): each affected block is recomputed in full over its written positions, ++// so the append-only stream self-corrects with NO ±inf init and NO merge. At decode only the ++// current (last) block is affected; at prefill all blocks are computed once. ++ ++#include "sparse-attn.cuh" ++ ++// opencoti #551 bug-744: load one K channel as f32 regardless of K's storage type. The fill now reads ++// k_cur (the NEW tokens, pre-quantization) — always f16 or f32 — so the kmin/kmax bound composes with ++// EVERY KV cache type (q8_0/q4_0/q6_0/turbo*/tcq*): the cache is quantized AFTER this, the bound is the ++// exact pre-quant envelope (Quest's upper bound is conservative either way). No dequant in the fill. ++template static __device__ __forceinline__ float sparse_ld(const kt * p, int d); ++template <> __device__ __forceinline__ float sparse_ld(const half * p, int d) { return __half2float(p[d]); } ++template <> __device__ __forceinline__ float sparse_ld(const float * p, int d) { return p[d]; } ++ ++template ++static __global__ void sparse_attn_fill_kernel( ++ const char * __restrict__ k_base, // k_cur (NEW tokens), f16/f32; column (p-p0) at k_base+(p-p0)*nb1 ++ const idx_t * __restrict__ k_idxs, // newly-written cache positions (I32 or I64); k_idxs[0] = p0 ++ half * __restrict__ kbounds, // [n_embd, n_block, 2] contiguous (persistent across steps) ++ const int n_embd, ++ const int n_new, // # of new-token columns present in k_base ++ const int n_block, ++ const int B, ++ const int64_t nb1) { // byte stride between k_cur columns (= k_cur->nb[2]) ++ const int b = blockIdx.x; ++ if (b >= n_block) { ++ return; ++ } ++ const int p0 = (int) k_idxs[0]; // first newly-written cache position ++ const int n_kv = p0 + n_new; // total cache length after this write ++ const int b_first = p0 / B; ++ // Append-only: blocks strictly below the first touched block are unchanged from a prior step. ++ if (b < b_first) { ++ return; ++ } ++ const int kv_lo = b * B; ++ int kv_hi = (b + 1) * B; ++ if (kv_hi > n_kv) { ++ kv_hi = n_kv; ++ } ++ if (kv_lo >= kv_hi) { ++ return; // block beyond the current cache length ++ } ++ // Only the NEW positions [max(kv_lo,p0), kv_hi) live in k_base; a block that already held tokens ++ // (kv_lo < p0, i.e. the partial last block at decode) MERGES into its prior kmin/kmax, a fully-new ++ // block OVERWRITES. min/max is exact so this reproduces the dense recompute byte-for-byte (f16 path). ++ const int new_lo = kv_lo < p0 ? p0 : kv_lo; ++ const bool straddle = kv_lo < p0; ++ ++ for (int d = threadIdx.x; d < n_embd; d += blockDim.x) { ++ float vmin = INFINITY; ++ float vmax = -INFINITY; ++ for (int p = new_lo; p < kv_hi; ++p) { ++ const kt * col = (const kt *) (k_base + (int64_t) (p - p0) * nb1); ++ const float v = sparse_ld(col, d); ++ vmin = fminf(vmin, v); ++ vmax = fmaxf(vmax, v); ++ } ++ const int64_t imin = (int64_t) b * n_embd + d; // w=0 (min) ++ const int64_t imax = ((int64_t) n_block + b) * n_embd + d; // w=1 (max) ++ if (straddle) { ++ kbounds[imin] = __float2half(fminf(__half2float(kbounds[imin]), vmin)); ++ kbounds[imax] = __float2half(fmaxf(__half2float(kbounds[imax]), vmax)); ++ } else { ++ kbounds[imin] = __float2half(vmin); ++ kbounds[imax] = __float2half(vmax); ++ } ++ } ++} ++ ++void ggml_cuda_sparse_attn_fill(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * k = dst->src[1]; // opencoti bug-744: k_cur (new tokens), NOT the cache view ++ const ggml_tensor * k_idxs = dst->src[2]; ++ ++ GGML_ASSERT(dst->type == GGML_TYPE_F16); ++ // opencoti bug-744: k_cur is the pre-quant K projection — f16 or f32 — so sparse composes with ALL ++ // cache KV types (the cache quantization happens downstream, in cpy_k). ++ GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32); ++ GGML_ASSERT(k_idxs->type == GGML_TYPE_I64 || k_idxs->type == GGML_TYPE_I32); ++ GGML_ASSERT(dst->ne[2] == 2); ++ ++ int B = 0; ++ memcpy(&B, dst->op_params, sizeof(int)); ++ GGML_ASSERT(B > 0); ++ ++ // opencoti #551 PERF-DIAG: LLAMA_SPARSE_NO_FILL skips the kmin/kmax update (kbounds goes stale — ++ // perf-bisection ONLY, not correctness). Isolates the fill cost from select on the decode path. ++ static const bool no_fill = getenv("LLAMA_SPARSE_NO_FILL") != nullptr; ++ if (no_fill) { return; } ++ ++ const int n_embd = dst->ne[0]; ++ const int n_block = dst->ne[1]; ++ // k_cur is [head_dim, n_head_kv, n_new, 1]; the new-token count is the position axis ne[2], and the ++ // per-token column stride is nb[2]. n_embd (= head_dim·n_head_kv) is the contiguous channel span. ++ const int n_new = k->ne[2]; ++ const int64_t nb1 = k->nb[2]; ++ ++ half * kb = (half *) dst->data; // dst aliases the kbounds buffer (cpy idiom) ++ ++ if (n_new <= 0 || n_block <= 0) { ++ return; ++ } ++ ++ const int nthreads = n_embd < 256 ? (n_embd < 32 ? 32 : n_embd) : 256; ++ const bool i64 = (k_idxs->type == GGML_TYPE_I64); ++ const bool kf16 = (k->type == GGML_TYPE_F16); ++ if (i64 && kf16) { ++ sparse_attn_fill_kernel<<>>( ++ (const char *) k->data, (const int64_t *) k_idxs->data, kb, n_embd, n_new, n_block, B, nb1); ++ } else if (i64) { ++ sparse_attn_fill_kernel<<>>( ++ (const char *) k->data, (const int64_t *) k_idxs->data, kb, n_embd, n_new, n_block, B, nb1); ++ } else if (kf16) { ++ sparse_attn_fill_kernel<<>>( ++ (const char *) k->data, (const int32_t *) k_idxs->data, kb, n_embd, n_new, n_block, B, nb1); ++ } else { ++ sparse_attn_fill_kernel<<>>( ++ (const char *) k->data, (const int32_t *) k_idxs->data, kb, n_embd, n_new, n_block, B, nb1); ++ } ++} ++ ++// --------------------------------------------------------------------------------------------- ++// GGML_OP_SPARSE_ATTN_SELECT — Quest block selector (B2). Two passes: ++// (1) score: per (block b, sequence s), score = max over query-heads h and rows of the Quest ++// upper bound Σ_d max(q_{h,d}·kmin_{d,b}, q_{h,d}·kmax_{d,b}) → global scratch. ++// (2) select: per sequence, top-K_SEL blocks by score (threshold bisection) OR the recent/sink ++// window → a SORTED, −1-padded list of selected block indices dst[s*n_block + i] (#551 ++// compacted list; the FA-VEC decode kernel walks it and stops at the first −1). ++// Union-over-heads/rows is exact for decode (one query row) and a safe superset for prefill. ++ ++#define SPARSE_SEL_NT 256 ++ ++static __global__ void sparse_attn_score_kernel( ++ const char * __restrict__ q_base, // FA query (f32), [head_dim, n_head, n_q, n_seq] ++ const half * __restrict__ kbounds, // [n_embd_k, n_block, 2] contiguous ++ float * __restrict__ scores, // [n_block, n_seq] out ++ const int * __restrict__ k_idxs, // #551 PERF: KV write indices (decode position), or nullptr ++ const int head_dim, ++ const int n_head, ++ const int n_q, ++ const int n_embd_k, ++ const int n_block, ++ const int n_sub, // #551 bug-743b: kbounds ROW count (= fill's ne[1]); the max ++ // plane lives at physical rows [n_sub, 2*n_sub). MUST match the ++ // fill's base, NOT the dynamic walk n_block (= ceil(n_kv/B)). ++ const int sub_per_block, // #551: kbounds sub-blocks per walk-block; score = max over them ++ const int refresh, // re-score every N decode tokens; skip the work otherwise ++ const int64_t nb1, const int64_t nb2, const int64_t nb3) { // q strides (bytes) ++ // #551 PERF (amortized): skip the (expensive) score computation on non-refresh decode steps — the select ++ // kernel reuses the cached list, so these scores are never read. Gate on the DEVICE position (k_idxs[0]) ++ // so it stays correct under CUDA-graph replay. Uniform return across the block. ++ if (k_idxs && refresh > 1) { ++ const int pos = k_idxs[0]; ++ if (pos % refresh != 0) { ++ return; ++ } ++ } ++ const int b = blockIdx.x; ++ const int s = blockIdx.y; ++ if (b >= n_block) { ++ return; ++ } ++ const int tid = threadIdx.x; ++ const int n_head_kv = n_embd_k / head_dim; ++ const int gqa = n_head / n_head_kv; ++ ++ __shared__ float sbuf[SPARSE_SEL_NT]; ++ __shared__ float s_score; ++ if (tid == 0) { ++ s_score = -INFINITY; ++ } ++ __syncthreads(); ++ ++ // #551 sub-block Quest bound: walk-block b owns kbounds sub-rows [b*SUB, b*SUB+SUB). The block score is ++ // the max over (head, query-row, sub-block) of the per-sub Quest upper bound. Finer sub-blocks tighten ++ // the kmin/kmax envelope so a needle's local sub-block scores high even when the 128-wide block is loose. ++ const int sub0 = b * sub_per_block; ++ const int sub1 = min(sub0 + sub_per_block, n_sub); ++ for (int h = 0; h < n_head; ++h) { ++ const int kvh = (gqa > 0) ? (h / gqa) : 0; ++ for (int qr = 0; qr < n_q; ++qr) { ++ const char * q_col = q_base + (int64_t) s*nb3 + (int64_t) qr*nb2 + (int64_t) h*nb1; ++ for (int sb = sub0; sb < sub1; ++sb) { ++ float partial = 0.0f; ++ for (int j = tid; j < head_dim; j += blockDim.x) { ++ const float qv = *(const float *) (q_col + (int64_t) j*4); ++ const int ch = kvh*head_dim + j; ++ const float fmin = __half2float(kbounds[(int64_t) sb *n_embd_k + ch]); ++ const float fmax = __half2float(kbounds[(int64_t)(n_sub + sb)*n_embd_k + ch]); ++ partial += fmaxf(qv*fmin, qv*fmax); ++ } ++ sbuf[tid] = partial; ++ __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { ++ if (tid < r) { sbuf[tid] += sbuf[tid + r]; } ++ __syncthreads(); ++ } ++ if (tid == 0) { s_score = fmaxf(s_score, sbuf[0]); } ++ __syncthreads(); ++ } ++ } ++ } ++ if (tid == 0) { ++ scores[(int64_t) s*n_block + b] = s_score; ++ } ++} ++ ++static __global__ void sparse_attn_select_kernel( ++ const float * __restrict__ scores, // [n_block, n_seq] ++ int * __restrict__ dst, // [n_list, n_seq] I32 sorted −1-padded selected-block list (#551) ++ const int * __restrict__ k_idxs, // #551 PERF: KV write indices (decode position), or nullptr ++ const int n_block, ++ const int n_list, // dst row stride/capacity (>= n_block; persistent buffer adds slack) ++ const int k_sel, ++ const int recent_blocks, ++ const int sink_blocks, ++ const int refresh, // re-select every N decode tokens; reuse the cached list otherwise ++ const int dbg) { // opencoti #551 bug-743 DIAG: print cnt/T/score-range when >0 ++ const int s = blockIdx.x; ++ const int tid = threadIdx.x; ++ ++ // #551 PERF (amortized selection): with a persistent buffer (k_idxs != nullptr) re-select only every ++ // `refresh` decode tokens — read the position from k_idxs[0] on the DEVICE so the decision stays correct ++ // under CUDA-graph replay (the launch args are captured, but device memory is re-read each replay). On a ++ // skipped step every thread returns uniformly, leaving the previously written list intact for FA-VEC. ++ if (k_idxs && refresh > 1) { ++ const int pos = k_idxs[0]; ++ if (pos % refresh != 0) { ++ return; ++ } ++ } ++ ++ const float * sc = scores + (int64_t) s*n_block; ++ int * out = dst + (int64_t) s*n_list; ++ ++ // init the whole list row to −1 (padding / FA-VEC walk terminator). Clearing the full capacity (n_list, ++ // not just n_block) guarantees stale entries past the emitted count are −1 even when the persistent ++ // buffer was sized for a larger ctx than the current n_block. ++ for (int b = tid; b < n_list; b += blockDim.x) { out[b] = -1; } ++ __syncthreads(); ++ ++ __shared__ float sbuf[SPARSE_SEL_NT]; ++ __shared__ float s_lo, s_hi, s_T; ++ __shared__ float s_scmin, s_scmax; // opencoti #551 bug-743 DIAG: pre-bisection score range ++ __shared__ int sibuf[SPARSE_SEL_NT]; ++ ++ // block min/max of scores ++ float lmin = INFINITY, lmax = -INFINITY; ++ for (int b = tid; b < n_block; b += blockDim.x) { ++ lmin = fminf(lmin, sc[b]); lmax = fmaxf(lmax, sc[b]); ++ } ++ sbuf[tid] = lmin; __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { if (tid < r) sbuf[tid] = fminf(sbuf[tid], sbuf[tid+r]); __syncthreads(); } ++ if (tid == 0) { s_lo = sbuf[0]; s_scmin = sbuf[0]; } ++ __syncthreads(); ++ sbuf[tid] = lmax; __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { if (tid < r) sbuf[tid] = fmaxf(sbuf[tid], sbuf[tid+r]); __syncthreads(); } ++ if (tid == 0) { s_hi = sbuf[0]; s_scmax = sbuf[0]; } ++ __syncthreads(); ++ ++ if (k_sel <= 0 || k_sel >= n_block) { ++ if (tid == 0) s_T = -INFINITY; // select-all anchor (topk==0 ⇒ inert, byte-identical) ++ } else { ++ // bisection: find threshold T so that #{b: score[b] >= T} ≈ k_sel ++ for (int it = 0; it < 30; ++it) { ++ __syncthreads(); ++ const float mid = 0.5f*(s_lo + s_hi); ++ int local = 0; ++ for (int b = tid; b < n_block; b += blockDim.x) { local += (sc[b] >= mid); } ++ sibuf[tid] = local; __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { if (tid < r) sibuf[tid] += sibuf[tid+r]; __syncthreads(); } ++ if (tid == 0) { if (sibuf[0] > k_sel) s_lo = mid; else s_hi = mid; } ++ } ++ if (tid == 0) s_T = s_lo; ++ } ++ __syncthreads(); ++ const float T = s_T; ++ ++ // emit a SORTED compacted list of kept block indices (single ordered pass; n_block is a few k and the ++ // select kernel is ~0-cost at decode). Ascending order ⇒ select-all reproduces the dense tile order, so ++ // topk=0 stays byte-identical with the dense FA-VEC path; the trailing −1 entries terminate the walk. ++ if (tid == 0) { ++ int cnt = 0; ++ for (int b = 0; b < n_block; ++b) { ++ const bool keep = (sc[b] >= T) || (b < sink_blocks) || (b >= n_block - recent_blocks); ++ if (keep) { out[cnt++] = b; } ++ } ++ // opencoti #551 bug-743 DIAG: emit the actual kept-count + threshold + score range for s==0. ++ if (dbg > 0 && s == 0) { ++ const int last = n_block > 0 ? n_block - 1 : 0; ++ printf("[sparse-sel s=0] n_block=%d k_sel=%d recent=%d sink=%d T=%g scmin=%g scmax=%g sc[0]=%g sc[mid]=%g sc[last]=%g => cnt=%d first=[%d %d %d %d] last_emit=%d\n", ++ n_block, k_sel, recent_blocks, sink_blocks, (double) T, (double) s_scmin, (double) s_scmax, ++ (double) sc[0], (double) sc[n_block/2], (double) sc[last], ++ cnt, cnt>0?out[0]:-9, cnt>1?out[1]:-9, cnt>2?out[2]:-9, cnt>3?out[3]:-9, cnt>0?out[cnt-1]:-9); ++ } ++ } ++} ++ ++// --------------------------------------------------------------------------------------------- ++// GGML_OP_SPARSE_ATTN_VSLASH — content-driven block selector (MInference-style, #551). ++// score(b) = max over heads h of Σ_{q in last q_win rows} Σ_{p in block b} softmax_h(q·K)[p] ++// i.e. the per-block attention MASS (sum of softmax probability that lands in the block), unioned ++// across query-heads by max. This is the MInference vertical-importance estimator, NOT the max ++// attention logit and NOT the Quest geometric spread bound. Both of those score a block by a MAX ++// over its keys — an extreme-value statistic whose distractor inflation grows with key count, so ++// the needle's lone constant-signal key gets buried as context grows and the lossless coverage ++// fraction RISES with ctx (the failure mode that sank Quest AND the max-logit vslash; bug-741). ++// Softmax is winner-take-all per query: block mass is bounded [0,1] and concentrates on the blocks ++// the query actually attends to, regardless of ctx length. One CUDA block per (query-head, seq) so ++// each head owns its softmax partition Z. The select kernel (top-K + recent/sink sorted compaction) ++// is reused VERBATIM. scores must be PRE-ZEROED (the launcher memsets) — heads accumulate by max. ++static __global__ void vslash_score_kernel( ++ const char * __restrict__ q_base, // FA query (f32), [head_dim, n_head, n_q, n_seq] ++ const char * __restrict__ k_base, // cache K (f16), column (p, kvh) at k_base + p*knb2 + kvh*knb1 ++ float * __restrict__ scores, // [n_block, n_seq] out, PRE-ZEROED; atomic-max accumulated ++ const int * __restrict__ k_idxs, // #551 PERF: KV write index (decode position), or nullptr ++ const int head_dim, ++ const int n_head, ++ const int n_q, ++ const int n_kv, ++ const int n_block, ++ const int B, // block size (keys per block) ++ const int gqa, // query heads per KV head (kvh = h / gqa) ++ const int q_win, // # of recent query rows that define the window ++ const int refresh, ++ const int64_t qnb1, const int64_t qnb2, const int64_t qnb3, // q strides (bytes) ++ const int64_t knb1, const int64_t knb2) { // k per-KV-head / per-position strides (bytes) ++ // #551 PERF (amortized): skip scoring on non-refresh decode steps — select reuses the cached list, ++ // so these scores are never read. Gate on the DEVICE position (k_idxs[0]) for CUDA-graph-replay safety. ++ if (k_idxs && refresh > 1) { ++ const int pos = k_idxs[0]; ++ if (pos % refresh != 0) { ++ return; ++ } ++ } ++ const int h = blockIdx.x; // one CUDA block per (query-head, sequence) ++ const int s = blockIdx.y; ++ if (h >= n_head) { ++ return; ++ } ++ const int tid = threadIdx.x; ++ const int kvh = (gqa > 0) ? (h / gqa) : 0; ++ ++ // dynamic shared: [0,n_block) = this-query block exp sums; [n_block,2*n_block) = cross-query accumulator ++ extern __shared__ float smem[]; ++ float * bsum_q = smem; ++ float * bsum_acc = smem + n_block; ++ for (int b = tid; b < n_block; b += blockDim.x) { ++ bsum_acc[b] = 0.0f; ++ } ++ __syncthreads(); ++ ++ __shared__ float sred[SPARSE_SEL_NT]; ++ __shared__ float s_M, s_Z; ++ ++ int qr0 = n_q - q_win; ++ if (qr0 < 0) { ++ qr0 = 0; ++ } ++ const char * q_head = q_base + (int64_t) s*qnb3 + (int64_t) h*qnb1; ++ ++ for (int qr = qr0; qr < n_q; ++qr) { ++ const char * qrow = q_head + (int64_t) qr*qnb2; ++ // CAUSAL bound: a prefill-window query at row qr sits at absolute position pmax = n_kv - n_q + qr ++ // (the chunk's queries occupy the last n_q cache positions). It may only attend keys p <= pmax — ++ // including future keys would pollute the estimate. At decode (n_q==1) pmax = n_kv-1 (all keys). ++ const int pmax = n_kv - n_q + qr; ++ for (int b = tid; b < n_block; b += blockDim.x) { ++ bsum_q[b] = 0.0f; ++ } ++ __syncthreads(); ++ ++ // Pass 1: M = max_p (q·K[p]) for softmax stability ++ float lmax = -INFINITY; ++ for (int p = tid; p <= pmax; p += blockDim.x) { ++ const half * kcol = (const half *) (k_base + (int64_t) p*knb2 + (int64_t) kvh*knb1); ++ float dot = 0.0f; ++ for (int d = 0; d < head_dim; ++d) { ++ dot += (*(const float *) (qrow + (int64_t) d*4)) * __half2float(kcol[d]); ++ } ++ lmax = fmaxf(lmax, dot); ++ } ++ sred[tid] = lmax; ++ __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { if (tid < r) sred[tid] = fmaxf(sred[tid], sred[tid+r]); __syncthreads(); } ++ if (tid == 0) { s_M = sred[0]; } ++ __syncthreads(); ++ const float M = s_M; ++ ++ // Pass 2: Z = Σ exp(dot-M); per-block bsum_q[b] += exp(dot-M) (causal: p <= pmax) ++ float lz = 0.0f; ++ for (int p = tid; p <= pmax; p += blockDim.x) { ++ const half * kcol = (const half *) (k_base + (int64_t) p*knb2 + (int64_t) kvh*knb1); ++ float dot = 0.0f; ++ for (int d = 0; d < head_dim; ++d) { ++ dot += (*(const float *) (qrow + (int64_t) d*4)) * __half2float(kcol[d]); ++ } ++ const float e = __expf(dot - M); ++ lz += e; ++ atomicAdd(&bsum_q[p / B], e); ++ } ++ sred[tid] = lz; ++ __syncthreads(); ++ for (int r = blockDim.x/2; r > 0; r >>= 1) { if (tid < r) sred[tid] += sred[tid+r]; __syncthreads(); } ++ if (tid == 0) { s_Z = sred[0]; } ++ __syncthreads(); ++ const float Z = s_Z > 0.0f ? s_Z : 1.0f; ++ ++ // normalize this query's block mass and fold into the cross-query accumulator ++ for (int b = tid; b < n_block; b += blockDim.x) { ++ bsum_acc[b] += bsum_q[b] / Z; ++ } ++ __syncthreads(); ++ } ++ ++ // union across heads: atomic-max this head's block mass into scores. Mass >= 0, so the IEEE ++ // int-reinterpretation is monotone and atomicMax on the int view is a valid float max. ++ for (int b = tid; b < n_block; b += blockDim.x) { ++ atomicMax((int *) &scores[(int64_t) s*n_block + b], __float_as_int(bsum_acc[b])); ++ } ++} ++ ++void ggml_cuda_sparse_attn_vslash(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * q = dst->src[0]; ++ const ggml_tensor * k_full = dst->src[1]; ++ ++ GGML_ASSERT(dst->type == GGML_TYPE_I32); ++ GGML_ASSERT(q->type == GGML_TYPE_F32); ++ GGML_ASSERT(k_full->type == GGML_TYPE_F16); ++ ++ int32_t p[7] = {0,0,0,0,0,1,1}; ++ memcpy(p, dst->op_params, sizeof(p)); ++ const int B = p[0]; ++ const int n_block = p[1]; ++ const int k_sel = p[2]; ++ const int recent_blocks = p[3]; ++ const int sink_blocks = p[4]; ++ const int refresh = p[5] < 1 ? 1 : p[5]; ++ const int q_win = p[6] < 1 ? 1 : p[6]; ++ ++ const int head_dim = q->ne[0]; ++ const int n_head = q->ne[1]; ++ const int n_q = q->ne[2]; ++ const int n_seq = q->ne[3]; ++ // get_k() returns K as [head_dim, n_head_kv, n_kv, 1]; the position axis is ne[2]/nb[2] (bug-743). ++ const int n_kv = (int) k_full->ne[2]; ++ const int n_head_kv = (int) k_full->ne[1]; ++ const int gqa = (n_head_kv > 0) ? (n_head / n_head_kv) : 1; ++ const int64_t knb1 = k_full->nb[1]; // per-KV-head stride ++ const int64_t knb2 = k_full->nb[2]; // per-position stride ++ ++ const int n_list = dst->ne[0]; ++ GGML_ASSERT(n_list >= n_block); ++ ++ const ggml_tensor * k_idxs = dst->src[2]; ++ const int * k_idxs_ptr = k_idxs ? (const int *) k_idxs->data : nullptr; ++ ++ if (n_block <= 0 || n_seq <= 0 || n_kv <= 0) { ++ return; ++ } ++ ++ ggml_cuda_pool_alloc scores(ctx.pool(), (size_t) n_block * n_seq); ++ // softmax block-mass accumulates across heads by atomic-max ⇒ scores must start at 0. ++ CUDA_CHECK(cudaMemsetAsync(scores.ptr, 0, (size_t) n_block * n_seq * sizeof(float), ctx.stream())); ++ ++ // one CUDA block per (query-head, seq); dynamic shared = 2·n_block floats (per-query + cross-query sums). ++ const dim3 grid_score(n_head, n_seq, 1); ++ const size_t smem = (size_t) 2 * n_block * sizeof(float); ++ vslash_score_kernel<<>>( ++ (const char *) q->data, (const char *) k_full->data, scores.ptr, k_idxs_ptr, ++ head_dim, n_head, n_q, n_kv, n_block, B, gqa, q_win, refresh, ++ q->nb[1], q->nb[2], q->nb[3], knb1, knb2); ++ ++ static const int sparse_dbg = getenv("OPENCOTI_SPARSE_DBG") ? atoi(getenv("OPENCOTI_SPARSE_DBG")) : 0; ++ sparse_attn_select_kernel<<>>( ++ scores.ptr, (int *) dst->data, k_idxs_ptr, n_block, n_list, k_sel, recent_blocks, sink_blocks, refresh, sparse_dbg); ++} ++ ++void ggml_cuda_sparse_attn_select(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * q = dst->src[0]; ++ const ggml_tensor * kbounds = dst->src[1]; ++ ++ GGML_ASSERT(dst->type == GGML_TYPE_I32); ++ GGML_ASSERT(q->type == GGML_TYPE_F32); ++ GGML_ASSERT(kbounds->type == GGML_TYPE_F16); ++ GGML_ASSERT(kbounds->ne[2] == 2); ++ ++ int32_t p[7] = {0,0,0,0,0,1,1}; ++ memcpy(p, dst->op_params, sizeof(p)); ++ const int n_block = p[1]; ++ const int k_sel = p[2]; ++ const int recent_blocks = p[3]; ++ const int sink_blocks = p[4]; ++ const int refresh = p[5] < 1 ? 1 : p[5]; ++ const int sub_per_block = p[6] < 1 ? 1 : p[6]; // #551 sub-block Quest bound (1 = plain B-block) ++ ++ const int head_dim = q->ne[0]; ++ const int n_head = q->ne[1]; ++ const int n_q = q->ne[2]; ++ const int n_seq = q->ne[3]; ++ const int n_embd_k = kbounds->ne[0]; ++ // #551: list row stride. Transient dst is [n_block, n_seq]; the persistent amortized buffer is ++ // [n_block_cap+1, n_seq] (capacity >= current n_block), so n_list >= n_block in general. ++ const int n_list = dst->ne[0]; ++ GGML_ASSERT(n_list >= n_block); ++ ++ // #551 PERF: persistent-buffer path attaches the KV write indices as src[2] (decode position source for ++ // the in-kernel refresh gate). nullptr ⇒ transient per-step selection (refresh forced to 1 in ggml.c). ++ const ggml_tensor * k_idxs = dst->src[2]; ++ const int * k_idxs_ptr = k_idxs ? (const int *) k_idxs->data : nullptr; ++ ++ if (n_block <= 0 || n_seq <= 0) { ++ return; ++ } ++ ++ ggml_cuda_pool_alloc scores(ctx.pool(), (size_t) n_block * n_seq); ++ ++ // #551 bug-743b: the max plane base is the kbounds ROW count (ne[1] = ceil(n_ctx/B_alloc)), which fill ++ // uses; the score previously read it at the dynamic walk n_block (= ceil(n_kv/B)) → max read landed in the ++ // MIN plane / garbage whenever n_kv < n_ctx (always), corrupting the Quest bound → near-random selection. ++ const int n_sub = (int) kbounds->ne[1]; ++ const dim3 grid_score(n_block, n_seq, 1); ++ sparse_attn_score_kernel<<>>( ++ (const char *) q->data, (const half *) kbounds->data, scores.ptr, k_idxs_ptr, ++ head_dim, n_head, n_q, n_embd_k, n_block, n_sub, sub_per_block, refresh, ++ q->nb[1], q->nb[2], q->nb[3]); ++ ++ // opencoti #551 bug-743 DIAG: OPENCOTI_SPARSE_DBG>0 prints the actual kept-count from the select kernel. ++ static const int sparse_dbg = getenv("OPENCOTI_SPARSE_DBG") ? atoi(getenv("OPENCOTI_SPARSE_DBG")) : 0; ++ sparse_attn_select_kernel<<>>( ++ scores.ptr, (int *) dst->data, k_idxs_ptr, n_block, n_list, k_sel, recent_blocks, sink_blocks, refresh, sparse_dbg); ++} +diff --git a/bk-t1/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cuh b/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cuh +new file mode 100644 +index 0000000..5cb7b0e +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/sparse-attn.cuh +@@ -0,0 +1,19 @@ ++#pragma once ++ ++// opencoti-hook: sparse-attn — #551 Quest block-selector. See docs/features/sparse_attn.md. ++#include "common.cuh" ++ ++// GGML_OP_SPARSE_ATTN_FILL forward: reduce the (already-written) cache K into per-block ++// channel-wise min/max, stored in the kbounds side-cache [n_embd_k, n_block, 2]. ++void ggml_cuda_sparse_attn_fill(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++ ++// GGML_OP_SPARSE_ATTN_SELECT forward: per block, score = max over query-heads/rows of the ++// Quest upper bound Σ_d max(q_d·kmin_d, q_d·kmax_d); then top-K_SEL blocks per sequence ++// (union over heads) OR recent/sink window → I32-packed bitmask dst[n_words, n_seq]. ++void ggml_cuda_sparse_attn_select(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++ ++// opencoti-hook: sparse-attn-vslash — #551 content-driven block selector. GGML_OP_SPARSE_ATTN_VSLASH ++// forward: per block, score = max over (last q_win query rows, key in block, head) of the actual logit ++// q·key (using cache K src[1], not a kbounds bound); then reuses the select kernel's top-K + recent/sink ++// sorted-compaction emit. See sparse-attn.cu. ++void ggml_cuda_sparse_attn_vslash(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +diff --git a/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh b/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh +index 5819745..b9cc5c0 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/tcq-decode-cb.cuh +@@ -22,3 +22,12 @@ void tcq3_set_decode_codebook_t3t2(const float * cb); // turbo3_tcq-K / turbo2_ + void tcq2_set_decode_codebook_t3t2(const float * cb); // turbo3_tcq-K / turbo2_tcq-V TU: its d_turbo2 (256) + void tcq3_set_decode_codebook_t2t3(const float * cb); // turbo2_tcq-K / turbo3_tcq-V TU: its d_turbo3 (512) + void tcq2_set_decode_codebook_t2t3(const float * cb); // turbo2_tcq-K / turbo3_tcq-V TU: its d_turbo2 (256) ++ ++// opencoti-hook: TCQ A′ — MMA-lift materialize codebook (#553/bug-710). The dequant-on-lift kernels ++// cpy_turbo{3,2}_tcq_f16 (turbo-cpy.cu) decode TCQ→f16 for the batched/MMA attention path (prefill + ++// spec-verify) using d_turbo{3,2}_tcq_cb_cpy — a THIRD per-TU static __constant__ copy that the decode ++// setters above DON'T reach. Defined in turbo-cpy.cu; set-rows.cu's TURBO_TCQ_CB / CB2 loader must call ++// these too, else the lift keeps the baked codebook while FA-VEC decode uses the override → draft(VEC) ++// vs verify(MMA-lift) decode the same KV bytes differently (acceptance collapse / KLD split). ++void tcq3_set_cpy_codebook(const float * cb); // 512 floats → turbo-cpy.cu d_turbo3_tcq_cb_cpy ++void tcq2_set_cpy_codebook(const float * cb); // 256 floats → turbo-cpy.cu d_turbo2_tcq_cb_cpy +diff --git a/bk-t1/llama.cpp/ggml/src/ggml-cuda/turbo-cpy-tcq-cb.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy-tcq-cb.cuh +new file mode 100644 +index 0000000..15f3eba +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy-tcq-cb.cuh +@@ -0,0 +1,110 @@ ++#pragma once ++// opencoti-hook: TCQ->f16 dequant-on-lift codebooks (#533/#505) — see docs/features/poly_kv.md. ++// VERBATIM byte-identical copy of the decode codebooks in fattn-tcq.cuh (extracted, not edited), ++// so the prefill-MMA materialize path (turbo-cpy.cu cpy_turbo{3,2}_tcq_f16) reconstructs the SAME ++// true-space values the fused FA-VEC decode path reads. Renamed *_cb_cpy to keep a private per-TU ++// __constant__ (the TURBO_TCQ_CB A'-override targets fattn-tcq.cuh's symbol only and does NOT reach ++// this TU; the shipped/default codebook is baked here — A'-sweep + prefill-MMA is a documented gap). ++// Regenerate (if codebook ever changes): .opencoti/m533-gen-tcq-cb.sh ++ ++static __constant__ float d_turbo3_tcq_cb_cpy[512] = { ++ -0.14559399f, -0.09062801f, -0.054925077f, -0.03699251f, -0.006363985f, +0.026264573f, +0.067378916f, +0.121981815f, ++ -0.18648055f, -0.106522456f, -0.052047577f, -0.011695214f, +0.021953275f, +0.059698727f, +0.09831437f, +0.16083933f, ++ -0.16390342f, -0.12639847f, -0.09513180f, -0.05938352f, -0.028396897f, +0.005973862f, +0.049104784f, +0.11334257f, ++ -0.25952467f, -0.079778515f, -0.036024813f, +0.0003641268f, +0.031858794f, +0.073280424f, +0.11835553f, +0.19738495f, ++ -0.14218009f, -0.10224814f, -0.062498566f, -0.027066832f, +0.00393002f, +0.04069300f, +0.08257346f, +0.14548601f, ++ -0.18673635f, -0.13438253f, -0.088401966f, -0.05205436f, -0.02032501f, +0.012399545f, +0.05127183f, +0.10316186f, ++ -0.10807011f, -0.065903045f, -0.032206114f, -0.0062006037f, +0.020679146f, +0.04422085f, +0.08313074f, +0.16821936f, ++ -0.22979105f, -0.14431947f, -0.07689272f, -0.02755307f, +0.009225173f, +0.046684854f, +0.08834142f, +0.13766693f, ++ -0.22114082f, -0.12612148f, -0.06890522f, -0.016128855f, +0.03691900f, +0.08474852f, +0.14940020f, +0.23229980f, ++ -0.14933491f, -0.099693604f, -0.06738499f, -0.037100967f, -0.009332986f, +0.023535024f, +0.060272533f, +0.109464675f, ++ -0.20200425f, -0.07398328f, -0.038700905f, -0.01714807f, +0.011161969f, +0.04528101f, +0.08902637f, +0.19573534f, ++ -0.16645233f, -0.124482535f, -0.089342155f, -0.04427387f, -0.007353691f, +0.028033108f, +0.066108435f, +0.15552913f, ++ -0.22295763f, -0.059887577f, -0.018804537f, +0.020141022f, +0.059682943f, +0.097920544f, +0.14080113f, +0.25698325f, ++ -0.14248224f, -0.089685425f, -0.050101686f, -0.017257255f, +0.011412255f, +0.040830314f, +0.07400172f, +0.11997315f, ++ -0.18649384f, -0.113997504f, -0.067775466f, -0.033394672f, +0.006586988f, +0.05312057f, +0.10433043f, +0.22344802f, ++ -0.16138338f, -0.108194515f, -0.07600300f, -0.05135381f, -0.023365447f, +0.0087320795f, +0.045431953f, +0.09113002f, ++ -0.12630440f, -0.07225349f, -0.032280035f, +0.0029231994f, +0.019239848f, +0.05081419f, +0.077840395f, +0.121695265f, ++ -0.08928155f, -0.044983763f, -0.009889568f, +0.020831043f, +0.05684458f, +0.09409702f, +0.13867535f, +0.19084482f, ++ -0.14182915f, -0.11380146f, -0.06904074f, -0.002002765f, +0.034864165f, +0.070399575f, +0.11403063f, +0.15394832f, ++ -0.10876417f, -0.056122433f, -0.02267638f, +0.011113975f, +0.039639056f, +0.074084364f, +0.10155376f, +0.12540291f, ++ -0.17693359f, -0.13940524f, -0.10049578f, -0.06796275f, -0.036915872f, +0.00062823476f, +0.042142134f, +0.17906062f, ++ -0.09253492f, -0.04290128f, -0.006311852f, +0.023908244f, +0.049849935f, +0.078770354f, +0.10818172f, +0.15166481f, ++ -0.12429565f, -0.07392063f, -0.029114135f, +0.0059440783f, +0.042675965f, +0.08425635f, +0.13836108f, +0.18634140f, ++ -0.11795639f, -0.07033707f, -0.034163877f, -0.0008773357f, +0.03334606f, +0.07188203f, +0.12216825f, +0.17097956f, ++ -0.18718453f, -0.14090346f, -0.097799584f, -0.059522875f, -0.019208657f, +0.03079176f, +0.09334672f, +0.15811224f, ++ -0.27198875f, -0.16546582f, -0.11433405f, -0.06933013f, -0.04026183f, -0.0061146915f, +0.029263576f, +0.07322499f, ++ -0.18471734f, -0.102074504f, -0.06492570f, -0.034418534f, -0.009636157f, +0.023043344f, +0.05751496f, +0.09905984f, ++ -0.22826399f, -0.15946552f, -0.09913176f, -0.06585259f, -0.03252090f, +0.001313243f, +0.03556729f, +0.21612854f, ++ -0.13243781f, -0.087299444f, -0.049820945f, -0.016216082f, +0.01799807f, +0.057916876f, +0.09001349f, +0.13221787f, ++ -0.19516511f, -0.120894566f, -0.076130204f, -0.051442243f, -0.029535033f, -0.0020043184f, +0.029452588f, +0.075566076f, ++ -0.27272871f, -0.15841717f, -0.105432935f, -0.06792948f, -0.024532158f, +0.014960791f, +0.054415092f, +0.101517834f, ++ -0.21153601f, -0.15015371f, -0.08676790f, -0.04414934f, -0.0042129597f, +0.033762872f, +0.07589151f, +0.12768789f, ++ -0.090428725f, -0.037582967f, +0.0013173596f, +0.03900247f, +0.06840049f, +0.116906695f, +0.16584939f, +0.25382105f, ++ -0.13446195f, -0.07865091f, -0.039625354f, -0.0028398742f, +0.03019514f, +0.06799379f, +0.11850997f, +0.17521496f, ++ -0.11350345f, -0.058599845f, -0.017512511f, +0.019431496f, +0.055897832f, +0.093173414f, +0.14820710f, +0.22092152f, ++ -0.15165758f, -0.08869354f, -0.04974287f, -0.01705474f, +0.013134752f, +0.04367713f, +0.07733791f, +0.12430801f, ++ -0.09329869f, -0.04673005f, -0.00045857552f, +0.042781368f, +0.07802363f, +0.11887439f, +0.16250038f, +0.28612965f, ++ -0.12571070f, -0.07786012f, -0.03843933f, -0.0075433915f, +0.025822964f, +0.066053316f, +0.12021536f, +0.18341768f, ++ -0.16079275f, -0.04921760f, -0.006114644f, +0.026215268f, +0.05699377f, +0.09813471f, +0.16080129f, +0.23786584f, ++ -0.09980837f, -0.048535258f, -0.0096120685f, +0.025387142f, +0.05979822f, +0.09875251f, +0.14474337f, +0.20324114f, ++ -0.15846540f, -0.09938028f, -0.061492465f, -0.03523542f, -0.0061364113f, +0.024916094f, +0.06037314f, +0.106796466f, ++ -0.20557843f, -0.123237535f, -0.07734871f, -0.044549115f, -0.017114898f, +0.01616654f, +0.049574375f, +0.092319444f, ++ -0.19221115f, -0.14642999f, -0.091701314f, -0.055265956f, -0.021026207f, +0.017720066f, +0.05786183f, +0.110154524f, ++ -0.09956386f, -0.03870283f, +0.003052007f, +0.034851722f, +0.06256365f, +0.09628840f, +0.13979156f, +0.16582295f, ++ -0.18026546f, -0.12448310f, -0.07424377f, -0.03954519f, -0.01221123f, +0.028641058f, +0.100819774f, +0.18240699f, ++ -0.21520759f, -0.15573645f, -0.09820838f, -0.051450998f, -0.012993679f, +0.021135861f, +0.058727216f, +0.105848536f, ++ -0.11207385f, -0.08335689f, -0.048542723f, -0.023198519f, +0.0039304253f, +0.037778318f, +0.07813917f, +0.13106476f, ++ -0.17849164f, -0.120988995f, -0.078016765f, -0.043093704f, -0.016565649f, +0.015182641f, +0.050754096f, +0.09595712f, ++ -0.22132620f, -0.13407415f, -0.065785654f, -0.013291034f, +0.032098345f, +0.07478225f, +0.12431934f, +0.19174045f, ++ -0.095454164f, -0.051898945f, -0.015116375f, -0.012596778f, +0.018636847f, +0.05006925f, +0.087654814f, +0.13754296f, ++ -0.15254061f, -0.09576059f, -0.052086458f, -0.01596074f, +0.017607626f, +0.04778498f, +0.08950204f, +0.14901252f, ++ -0.26057002f, -0.12472382f, -0.074396215f, -0.03764066f, +0.0011168446f, +0.061569117f, +0.10793752f, +0.19771695f, ++ -0.08661132f, -0.045195263f, -0.016098704f, +0.012780116f, +0.040476497f, +0.074102715f, +0.074102715f, +0.12635531f, ++ -0.14047913f, -0.059587404f, -0.016261123f, +0.019801628f, +0.053541403f, +0.096650146f, +0.15005490f, +0.21051759f, ++ -0.22986396f, -0.11964334f, -0.07266585f, -0.026522418f, +0.018169926f, +0.058630653f, +0.100647695f, +0.15919648f, ++ -0.13251697f, -0.077567816f, -0.042766172f, -0.011389967f, +0.01831755f, +0.05304656f, +0.09620367f, +0.15567583f, ++ -0.119819686f, -0.06772876f, -0.028123451f, +0.00876240f, +0.014405836f, +0.048829112f, +0.08422175f, +0.13823749f, ++ -0.16379014f, -0.08956941f, -0.041652776f, +0.008921398f, +0.05473602f, +0.10037984f, +0.16022855f, +0.23457925f, ++ -0.115844205f, -0.05939626f, -0.020390417f, +0.01374377f, +0.044976473f, +0.07873563f, +0.12207942f, +0.18412720f, ++ -0.19048831f, -0.07587487f, -0.03220580f, -0.00011795067f, +0.02721784f, +0.04380719f, +0.07886723f, +0.13193911f, ++ -0.13935551f, -0.092902906f, -0.052706074f, -0.017797327f, +0.015312965f, +0.056098964f, +0.11203423f, +0.24448302f, ++ -0.17986591f, -0.10738580f, -0.06376371f, -0.026595421f, +0.00842492f, +0.04272362f, +0.08608052f, +0.15240218f, ++ -0.10953678f, -0.057022586f, -0.012483291f, +0.024463262f, +0.06076792f, +0.09776234f, +0.12983681f, +0.18648379f, ++ -0.16471463f, -0.089491285f, -0.037574016f, +0.004444791f, +0.039293647f, +0.07845859f, +0.12893885f, +0.23508036f ++}; ++ ++static __constant__ float d_turbo2_tcq_cb_cpy[256] = { ++ -0.18030643f, -0.11009848f, -0.04742626f, +0.02894132f, -0.10523465f, -0.031312924f, +0.031491395f, +0.12263535f, ++ -0.15660362f, -0.055477407f, +0.0046675834f, +0.06166081f, -0.07506216f, -0.016963918f, +0.043737844f, +0.116496615f, ++ -0.08632783f, -0.022493735f, +0.041032985f, +0.10660284f, -0.06274858f, -0.0036939639f, +0.02095157f, +0.07539709f, ++ -0.09802641f, -0.008419088f, +0.059072323f, +0.17311879f, -0.093109086f, -0.02654333f, +0.014827672f, +0.07793592f, ++ -0.031235758f, +0.01271591f, +0.08752262f, +0.17246453f, -0.14595252f, -0.07227624f, +0.013628688f, +0.08131674f, ++ -0.036909282f, +0.0018896917f, +0.05209119f, +0.12407892f, -0.13689458f, -0.06054520f, +0.0064648795f, +0.07551241f, ++ -0.18980840f, -0.110128626f, -0.046503957f, +0.026387159f, -0.034967307f, +0.04810357f, +0.072072044f, +0.14355458f, ++ -0.10182410f, -0.02907887f, +0.014033012f, +0.083419636f, -0.056140676f, +0.008405868f, +0.066070884f, +0.14037225f, ++ -0.117427245f, -0.047159385f, +0.016928354f, +0.08142885f, -0.029359628f, +0.045608785f, +0.10559447f, +0.20061271f, ++ -0.040425077f, +0.029068163f, +0.08408973f, +0.13628258f, -0.16633821f, -0.10711727f, -0.04196669f, +0.027895834f, ++ -0.0054065837f, +0.058898676f, +0.12688550f, +0.18268861f, -0.16287325f, -0.11218357f, -0.07165227f, -0.009524379f, ++ -0.24026902f, -0.073219374f, -0.0005165726f, +0.05959821f, -0.05532953f, +0.027044486f, +0.09425678f, +0.15356481f, ++ -0.14381111f, -0.10563502f, -0.037867088f, +0.023611993f, -0.03624307f, +0.049588434f, +0.12192037f, +0.23462485f, ++ -0.14990251f, -0.09659304f, -0.05886742f, +0.014878461f, -0.009889551f, +0.06910514f, +0.12120181f, +0.22596690f, ++ -0.08290075f, -0.009009629f, +0.066151775f, +0.12188313f, -0.11591514f, -0.06952189f, -0.031633306f, +0.023740824f, ++ -0.20510401f, -0.103369795f, +0.09148037f, +0.17268716f, -0.16597997f, -0.09207068f, -0.032810967f, +0.024847647f, ++ -0.02487482f, +0.049298953f, +0.09624215f, +0.14217524f, -0.18418685f, -0.10147012f, -0.05841265f, +0.008057022f, ++ -0.14269894f, -0.092456274f, -0.026881337f, +0.049792137f, -0.019881032f, +0.030333601f, +0.09736802f, +0.17764080f, ++ -0.19579841f, -0.114739306f, -0.026823774f, +0.07466014f, -0.09001050f, -0.041468445f, +0.028473806f, +0.08870695f, ++ -0.019396419f, +0.042828932f, +0.10885327f, +0.13335012f, -0.15005013f, -0.074581385f, -0.028608415f, +0.03848942f, ++ -0.09687270f, -0.057059396f, +0.0077843578f, +0.06302297f, -0.23247094f, -0.14509225f, -0.032651436f, +0.027010715f, ++ -0.047595482f, +0.06280303f, +0.114691675f, +0.17124057f, -0.21092793f, -0.13704823f, -0.07340412f, +0.0039013291f, ++ -0.062834196f, +0.012601906f, +0.012601906f, +0.08721347f, -0.13256435f, -0.024173854f, +0.07723171f, +0.14801070f, ++ -0.06471605f, -0.0017903054f, -0.0017903054f, +0.058302354f, -0.09731802f, -0.03400696f, +0.02762442f, +0.08986137f, ++ -0.08288722f, -0.019051429f, +0.045709886f, +0.15211061f, -0.09507891f, -0.015612489f, +0.025347246f, +0.087257534f, ++ -0.066236064f, -0.0047936034f, +0.06386274f, +0.15401669f, -0.105809286f, -0.051802177f, +0.01073050f, +0.08292137f, ++ -0.11884470f, -0.04404144f, +0.02550729f, +0.02550729f, -0.01731189f, +0.062161792f, +0.12127554f, +0.21981733f, ++ -0.17066145f, -0.11660990f, -0.049425896f, +0.021293938f, -0.04711412f, +0.026577346f, +0.055197213f, +0.12541275f, ++ -0.028268812f, +0.015206398f, +0.09002519f, +0.12699963f, -0.10059831f, -0.026676945f, +0.059903253f, +0.13054545f, ++ -0.09582803f, -0.033371232f, +0.010346129f, +0.066766635f, -0.09964944f, -0.028686784f, +0.021184925f, +0.09120017f, ++ -0.16957201f, -0.07594450f, +0.04172865f, +0.18313301f, -0.051526368f, +0.011877304f, +0.011877304f, +0.07956263f, ++ -0.13432936f, -0.05269006f, +0.03536416f, +0.117640756f, -0.022776067f, +0.042032316f, +0.10472976f, +0.18042557f ++}; +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu +index cf7a5dc..41611a4 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-cpy.cu +@@ -10,6 +10,20 @@ + #include "common.cuh" + #include "turbo-dequant.cuh" // TURBO{3}_WHT_SIGNS1/2 + TURBO{2,3,4}_CENTROIDS tables (this TU's copy) + #include "turbo-cpy.cuh" ++#include "turbo-cpy-tcq-cb.cuh" // opencoti-hook #533: TCQ decode codebooks (this TU's copy, == fattn-tcq.cuh) ++#include "tcq-decode-cb.cuh" // opencoti-hook #553: declares tcq{3,2}_set_cpy_codebook (defined below) ++ ++// opencoti-hook: TCQ A′ — MMA-lift codebook override (#553/bug-710). d_turbo{3,2}_tcq_cb_cpy are static ++// __constant__ in THIS TU (turbo-cpy-tcq-cb.cuh); cudaMemcpyToSymbol on a static symbol only reaches the ++// in-TU copy, so the TURBO_TCQ_CB / CB2 runtime override (set-rows.cu) must call these to apply to the ++// dequant-on-lift path, mirroring the FA-VEC tcq{3,2}_set_decode_codebook* setters. Without it the lift ++// (prefill + spec-verify) keeps the baked codebook while FA-VEC decode honors the override → mismatch. ++void tcq3_set_cpy_codebook(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo3_tcq_cb_cpy, cb, 512 * sizeof(float))); ++} ++void tcq2_set_cpy_codebook(const float * cb) { ++ CUDA_CHECK(cudaMemcpyToSymbol(d_turbo2_tcq_cb_cpy, cb, 256 * sizeof(float))); ++} + + // Cooperative inverse WHT over shared buf[128] (already holding centroid×norm in rotated space), + // then InnerQ ×scale_inv and a strided f16 write of element e0+t. All 128 threads of the block +@@ -108,6 +122,40 @@ static __global__ void cpy_turbo8_f16(TURBO_CPY_PARAMS) { + turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, scale_inv, head_dim); + } + ++// opencoti-hook: TCQ→f16 dequant-on-lift (#533/#505) — prefill-MMA fix + DCA-compose. ++// TCQ was fused-only (no f16 materialize), forcing prefill through FA-VEC (no tensor cores) ++// and aborting under DCA's f16-lift. These two kernels mirror cpy_turbo3_f16 exactly but decode ++// the trellis bitstream via the 512/256-entry codebook (byte-identical to fattn-tcq.cuh's reader), ++// then the SAME cooperative inverse-FWHT recovers TRUE-space f16. TCQ carries NO InnerQ, so ++// scale_inv=nullptr. norm is used raw — the per-side α (K×1.0/V×1.04) is already baked into norm at ++// encode (turbo-tcq-cuda.cuh:359) and the decode-time α defaults to 1.0 (no double-apply), so this ++// matches the FA-VEC reader at the shipped codebook/α. Bit extract == fattn-tcq.cuh:150-152 / 205-210. ++static __global__ void cpy_turbo3_tcq_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo3_tcq); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ const int bp = t * 3; ++ const uint16_t raw = (uint16_t) blk->qs[bp / 8] | ((uint16_t) blk->qs[bp / 8 + 1] << 8); ++ buf[t] = d_turbo3_tcq_cb_cpy[(raw >> (bp % 8)) & 0x1FF] * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, nullptr, head_dim); ++} ++ ++static __global__ void cpy_turbo2_tcq_f16(TURBO_CPY_PARAMS) { ++ const int t = threadIdx.x; ++ const int64_t e0 = (int64_t) blockIdx.x * 128; ++ if (e0 >= ne) return; ++ TURBO_SRC_BLOCK(block_turbo2_tcq); ++ const float norm = __half2float(blk->norm); ++ __shared__ float buf[128]; ++ const int bp = t * 2; ++ const uint16_t raw = (uint16_t) blk->qs[bp / 8] | ((uint16_t) blk->qs[bp / 8 + 1] << 8); ++ buf[t] = d_turbo2_tcq_cb_cpy[(raw >> (bp % 8)) & 0xFF] * norm; ++ turbo_wht_inv_write(buf, t, e0, cdst, ne10, ne11, ne12, nb10, nb11, nb12, nb13, nullptr, head_dim); ++} ++ + void ggml_cpy_turbo_f16_cuda( + enum ggml_type src_type, const char * cx, char * cdst, int64_t ne, + int64_t ne00, int64_t ne01, int64_t ne02, +@@ -127,6 +175,8 @@ void ggml_cpy_turbo_f16_cuda( + case GGML_TYPE_TURBO4_0: LAUNCH(cpy_turbo4_f16); break; + case GGML_TYPE_TURBO2_0: LAUNCH(cpy_turbo2_f16); break; + case GGML_TYPE_TURBO8_0: LAUNCH(cpy_turbo8_f16); break; ++ case GGML_TYPE_TURBO3_TCQ: LAUNCH(cpy_turbo3_tcq_f16); break; // opencoti-hook #533 ++ case GGML_TYPE_TURBO2_TCQ: LAUNCH(cpy_turbo2_tcq_f16); break; // opencoti-hook #533 + default: GGML_ABORT("ggml_cpy_turbo_f16_cuda: non-turbo src_type %d\n", (int) src_type); + } + #undef LAUNCH +diff --git a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +index 887aac2..3f38789 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/turbo-tcq-cuda.cuh +@@ -202,8 +202,14 @@ static __global__ void __launch_bounds__(TCQ3_WPB*32, 3) k_set_rows_turbo3_tcq( + // each group's 8 predecessors (g*8..g*8+7) fall entirely within lane g/2, so the predecessor-min is + // thread-local (no shuffle). A 64-entry per-warp smem broadcast under __syncwarp replaces the block barrier. + acc_t c[16]; ++ // opencoti-hook: L3 (#541) decode-tps — hoist the loop-invariant TCQ codebook reads out of the 128-step ++ // Viterbi DP. d_turbo3_tcq_codebook[16*lane+i] depends only on (lane,i), never on t, yet the warp-sync ++ // design re-read it from __constant__ every timestep with 32-way per-warp serialization (the #534 ++ // anti-pattern, here in the encode). Loading each lane's 16 entries into registers ONCE is byte-identical ++ // (same loads, just hoisted) and removes 127×16 serialized constant reads/group. ++ acc_t cb[16]; + #pragma unroll +- for (int i = 0; i < 16; i++) c[i] = 0.0f; ++ for (int i = 0; i < 16; i++) { c[i] = 0.0f; cb[i] = d_turbo3_tcq_codebook[16*lane + i]; } + for (int t = 0; t < 128; t++) { + acc_t pm0 = c[0]; int pp0 = 0; + #pragma unroll +@@ -216,7 +222,7 @@ static __global__ void __launch_bounds__(TCQ3_WPB*32, 3) k_set_rows_turbo3_tcq( + __syncwarp(); + acc_t xt = x[t]; + #pragma unroll +- for (int i = 0; i < 16; i++) { int s = 16*lane + i; acc_t d = xt - d_turbo3_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; } ++ for (int i = 0; i < 16; i++) { int s = 16*lane + i; acc_t d = xt - cb[i]; c[i] = pred_min_cost[s & 0x3F] + d * d; } + __syncwarp(); + } + // --- argmin over 512 states (per-lane min over 16, then warp-shuffle) --- +@@ -427,8 +433,9 @@ static __global__ void __launch_bounds__(TCQ2_WPB*32, 3) k_set_rows_turbo2_tcq( + // lane L owns states {8L..8L+7} = trellis-groups 2L (states 8L..8L+3) and 2L+1 (states 8L+4..8L+7); each + // group's 4 predecessors (g*4..g*4+3) fall within lane g/2, so predecessor-min is thread-local (no shuffle). + acc_t c[8]; ++ acc_t cb[8]; // opencoti-hook: L3 (#541) — hoist loop-invariant codebook out of the Viterbi DP (see turbo3 note above) + #pragma unroll +- for (int i = 0; i < 8; i++) c[i] = 0.0f; ++ for (int i = 0; i < 8; i++) { c[i] = 0.0f; cb[i] = d_turbo2_tcq_codebook[8*lane + i]; } + for (int t = 0; t < 128; t++) { + acc_t pm0 = c[0]; int pp0 = 0; + #pragma unroll +@@ -441,7 +448,7 @@ static __global__ void __launch_bounds__(TCQ2_WPB*32, 3) k_set_rows_turbo2_tcq( + __syncwarp(); + acc_t xt = x[t]; + #pragma unroll +- for (int i = 0; i < 8; i++) { int s = 8*lane + i; acc_t d = xt - d_turbo2_tcq_codebook[s]; c[i] = pred_min_cost[s & 0x3F] + d * d; } ++ for (int i = 0; i < 8; i++) { int s = 8*lane + i; acc_t d = xt - cb[i]; c[i] = pred_min_cost[s & 0x3F] + d * d; } + __syncwarp(); + } + // --- argmin over 256 states (per-lane min over 8, then warp-shuffle) --- +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +index 2defaae..261d69e 100644 +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -1158,9 +1158,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { + "FLASH_ATTN_TAIL_PARTIAL", + "TURBO_WHT", + "FLASH_ATTN_EXT_LSE", ++ "SPARSE_ATTN_FILL", ++ "SPARSE_ATTN_SELECT", ++ "SPARSE_ATTN_VSLASH", + }; + +-static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); ++static_assert(GGML_OP_COUNT == 104, "GGML_OP_COUNT != 104"); + + static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "none", +@@ -1273,9 +1276,12 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { + "flash_attn_tail_partial(q,k,v)", + "turbo_wht(x)", + "flash_attn_ext_lse(q,k,v)", ++ "sparse_attn_fill(kb,k)", ++ "sparse_attn_select(q,kb)", ++ "sparse_attn_vslash(q,k)", + }; + +-static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); ++static_assert(GGML_OP_COUNT == 104, "GGML_OP_COUNT != 104"); + + static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); + +@@ -5821,6 +5827,136 @@ struct ggml_tensor * ggml_turbo_wht( + return result; + } + ++// opencoti-hook: sparse-attn — #551 Quest per-block kmin/kmax fill. See docs/features/sparse_attn.md. ++struct ggml_tensor * ggml_sparse_attn_fill( ++ struct ggml_context * ctx, ++ struct ggml_tensor * kbounds, ++ struct ggml_tensor * k_cur, ++ struct ggml_tensor * k_idxs, ++ int block_size) { ++ GGML_ASSERT(kbounds->type == GGML_TYPE_F16); ++ GGML_ASSERT(kbounds->ne[2] == 2); // [n_embd_k, n_block, 2] : [..,0]=min, [..,1]=max ++ GGML_ASSERT(block_size > 0); ++ ++ // Write in-place into the kbounds side-cache (ggml_cpy idiom): result aliases kbounds' ++ // buffer so the SPARSE_ATTN_FILL node carries the write edge that orders the selector ++ // after the fill. src[0]=kbounds doubles as the existing values to merge (incremental). ++ struct ggml_tensor * result = ggml_view_tensor(ctx, kbounds); ++ ++ result->op = GGML_OP_SPARSE_ATTN_FILL; ++ result->src[0] = kbounds; ++ result->src[1] = k_cur; ++ result->src[2] = k_idxs; ++ ++ memcpy(result->op_params, &block_size, sizeof(int)); ++ ++ return result; ++} ++ ++struct ggml_tensor * ggml_sparse_attn_select( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * kbounds, ++ struct ggml_tensor * block_sel, ++ struct ggml_tensor * k_idxs, ++ int block_size, ++ int n_block, ++ int k_sel, ++ int recent_blocks, ++ int sink_blocks, ++ int refresh, ++ int sub_per_block) { ++ GGML_ASSERT(kbounds->type == GGML_TYPE_F16); ++ GGML_ASSERT(kbounds->ne[2] == 2); ++ GGML_ASSERT(block_size > 0); ++ GGML_ASSERT(n_block > 0); ++ GGML_ASSERT(sub_per_block >= 1); ++ ++ // dst: #551 compacted SORTED, −1-padded list of selected block indices per sequence (q->ne[3]); union ++ // over KV-heads and query rows. FA-VEC iterates the list (block_sel[seq*stride + i]) and stops at the ++ // first −1 — O(selected), not O(capacity). ++ const int64_t n_seq = q->ne[3]; ++ struct ggml_tensor * result; ++ if (block_sel) { ++ // #551 PERF (amortized): write IN-PLACE into the persistent per-stream buffer (cpy idiom, like ++ // SPARSE_ATTN_FILL). block_sel is [n_block+1, n_seq] I32 (capacity n_block + a reserved scratch ++ // slot). The kernel re-selects only every `refresh` decode tokens (position read from k_idxs[0]), ++ // reusing the cached list otherwise — keeps CUDA-graph topology stable (no add/remove of nodes). ++ GGML_ASSERT(block_sel->type == GGML_TYPE_I32); ++ GGML_ASSERT(block_sel->ne[0] >= n_block); ++ GGML_ASSERT(k_idxs); ++ GGML_ASSERT(refresh >= 1); ++ result = ggml_view_tensor(ctx, block_sel); ++ result->src[2] = k_idxs; ++ } else { ++ // legacy transient per-step list [n_block, n_seq] (multi-stream fallback; selects every step). ++ result = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_block, n_seq); ++ refresh = 1; ++ } ++ ++ result->op = GGML_OP_SPARSE_ATTN_SELECT; ++ result->src[0] = q; ++ result->src[1] = kbounds; ++ ++ int32_t params[7] = { block_size, n_block, k_sel, recent_blocks, sink_blocks, refresh, sub_per_block }; ++ memcpy(result->op_params, params, sizeof(params)); ++ ++ return result; ++} ++ ++// opencoti-hook: sparse-attn-vslash — #551 content-driven block selector. Unlike the Quest selector ++// (geometric per-dim kmin/kmax spread bound, which over-ranks high-spread filler and DEGRADES with ++// context), vslash scores each block by the EXACT per-block max attention logit over a window of ++// recent queries: score(b) = max over (q in last q_win rows, key p in block b, head h) of (q_h · key_p). ++// This is the "vertical" importance of MInference (arXiv:2407.02490): the question's own attention ++// picks the needle's key column. Reuses the select kernel's top-K + recent/sink compaction verbatim; ++// only the score source differs (actual cache K, src[1], instead of kbounds). op_params = ++// {block_size, n_block, k_sel, recent_blocks, sink_blocks, refresh, q_win}. ++struct ggml_tensor * ggml_sparse_attn_vslash( ++ struct ggml_context * ctx, ++ struct ggml_tensor * q, ++ struct ggml_tensor * k_full, ++ struct ggml_tensor * block_sel, ++ struct ggml_tensor * k_idxs, ++ int block_size, ++ int n_block, ++ int k_sel, ++ int recent_blocks, ++ int sink_blocks, ++ int refresh, ++ int q_win) { ++ GGML_ASSERT(k_full->type == GGML_TYPE_F16); ++ GGML_ASSERT(block_size > 0); ++ GGML_ASSERT(n_block > 0); ++ GGML_ASSERT(q_win >= 1); ++ ++ const int64_t n_seq = q->ne[3]; ++ struct ggml_tensor * result; ++ if (block_sel) { ++ // #551 PERF (amortized): write IN-PLACE into the persistent per-stream buffer (cpy idiom, ++ // mirror of SPARSE_ATTN_SELECT). The kernel re-selects only every `refresh` decode tokens ++ // (position read from k_idxs[0] on the device), reusing the cached list otherwise. ++ GGML_ASSERT(block_sel->type == GGML_TYPE_I32); ++ GGML_ASSERT(block_sel->ne[0] >= n_block); ++ GGML_ASSERT(k_idxs); ++ GGML_ASSERT(refresh >= 1); ++ result = ggml_view_tensor(ctx, block_sel); ++ result->src[2] = k_idxs; ++ } else { ++ result = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_block, n_seq); ++ refresh = 1; ++ } ++ ++ result->op = GGML_OP_SPARSE_ATTN_VSLASH; ++ result->src[0] = q; ++ result->src[1] = k_full; ++ ++ int32_t params[7] = { block_size, n_block, k_sel, recent_blocks, sink_blocks, refresh, q_win }; ++ memcpy(result->op_params, params, sizeof(params)); ++ ++ return result; ++} ++ + void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec) { +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index b5be5f5..a0a435c 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -381,6 +381,17 @@ extern "C" { + bool dca_enabled; + uint32_t dca_chunk_size; + float dca_yarn_factor; ++ // opencoti #551 sparse-attn — Quest-style block-selector sparse attention (docs/features/sparse_attn.md) ++ // sparse_attn_enabled: false = off (default). block_size: KV-block granularity (B_SEL, 64). ++ // topk: # blocks kept per query (0 = all/off, 0xFFFFFFFF = auto ¼ floor 64). recent: ++ // forced-recent window tokens (0 = none). sink: leading sink blocks always kept (default 1). ++ bool sparse_attn_enabled; ++ uint32_t sparse_attn_block_size; ++ uint32_t sparse_attn_topk; ++ uint32_t sparse_attn_recent; ++ uint32_t sparse_attn_sink; ++ uint32_t sparse_attn_refresh; // opencoti #551 PERF — re-select sparse blocks every N decode tokens (amortize selection machinery) ++ uint32_t sparse_attn_mode; // opencoti-hook: sparse-attn-vslash — #551 0 = Quest bound (default), 1 = vertical-slash content-driven + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 2526f84..2d87414 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -114,6 +114,14 @@ llama_context::llama_context( + cparams.dca_enabled = params.dca_enabled; + cparams.dca_chunk_size = params.dca_chunk_size; + cparams.dca_yarn_factor = params.dca_yarn_factor; ++ // opencoti #551 sparse-attn — see docs/features/sparse_attn.md ++ cparams.sparse_attn_enabled = params.sparse_attn_enabled; ++ cparams.sparse_attn_block_size = params.sparse_attn_block_size; ++ cparams.sparse_attn_topk = params.sparse_attn_topk; ++ cparams.sparse_attn_recent = params.sparse_attn_recent; ++ cparams.sparse_attn_sink = params.sparse_attn_sink; ++ cparams.sparse_attn_refresh = params.sparse_attn_refresh < 1 ? 1 : params.sparse_attn_refresh; ++ cparams.sparse_attn_mode = params.sparse_attn_mode; // opencoti-hook: sparse-attn-vslash — #551 + + cparams.n_threads = params.n_threads; + cparams.n_threads_batch = params.n_threads_batch; +@@ -4229,6 +4237,13 @@ llama_context_params llama_context_default_params() { + /*.dca_enabled =*/ false, // opencoti F5 dca — off by default + /*.dca_chunk_size =*/ 0, // opencoti F5 dca — 0 = auto (<- n_ctx_orig_yarn) + /*.dca_yarn_factor =*/ 1.0f, // opencoti F5 dca — 1.0 = no YaRN stretch ++ /*.sparse_attn_enabled =*/ false, // opencoti #551 sparse-attn — off by default ++ /*.sparse_attn_block_size =*/ 64, // opencoti #551 sparse-attn — KV-block granularity B_SEL ++ /*.sparse_attn_topk =*/ 0, // opencoti #551 sparse-attn — 0 = all blocks (off) ++ /*.sparse_attn_recent =*/ 0, // opencoti #551 sparse-attn — 0 = no forced-recent window ++ /*.sparse_attn_sink =*/ 1, // opencoti #551 sparse-attn — leading sink blocks always kept ++ /*.sparse_attn_refresh =*/ 8, // opencoti #551 PERF — re-select every N decode tokens (amortize) ++ /*.sparse_attn_mode =*/ 0, // opencoti-hook: sparse-attn-vslash — #551 0 = Quest (default) + /*.n_rs_seq =*/ 0, + /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default + /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +index 1aa52a5..41608b8 100644 +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -56,6 +56,16 @@ struct llama_cparams { + bool dca_enabled; + uint32_t dca_chunk_size; + float dca_yarn_factor; ++ // opencoti #551 sparse-attn — Quest-style block-selector (docs/features/sparse_attn.md). ++ // sparse_attn_enabled: per-query top-K KV-block selection before QK (false = off, default). ++ // sparse_attn_block_size: KV-block granularity B_SEL. topk: # blocks kept (0 = all). recent: forced window. ++ bool sparse_attn_enabled; ++ uint32_t sparse_attn_block_size; ++ uint32_t sparse_attn_topk; ++ uint32_t sparse_attn_recent; ++ uint32_t sparse_attn_sink; ++ uint32_t sparse_attn_refresh; // #551 PERF: re-run the selector every N decode tokens (amortize; 1 = every step) ++ uint32_t sparse_attn_mode; // opencoti-hook: sparse-attn-vslash — #551 0 = Quest bound, 1 = vertical-slash content-driven + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index 2fe40de..e0d5e2a 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -22,6 +22,12 @@ + #include + #include + ++// opencoti-hook: sparse-attn #551 — carries the Quest block-selector bitmask from build_attn's ++// store block (where the fill result is available, so the read edge orders select-after-fill) ++// into build_attn_mha, which attaches it as the flash_attn_ext op's src[7]. Set tightly around ++// the standard build_attn_mha call and cleared after, so it never leaks into other FA paths. ++static thread_local ggml_tensor * opencoti_sparse_block_sel = nullptr; ++ + // dedup helpers + + static ggml_tensor * build_attn_inp_kq_mask( +@@ -2094,24 +2100,26 @@ ggml_tensor * llm_graph_context::build_attn_mha( + const bool fused_turbo_512 = + k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0 + || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ); // opencoti TCQ #447/#500 (Gemma-4 GLOBAL) +- // opencoti-hook: TCQ FA-VEC decode (#447/#500). TCQ has NO materialize-lift (turbo-cpy.cu has no +- // TCQ→f16 inverse-FWHT cast yet), so it must ALWAYS fuse — prefill (large n_q) rides the VEC-tiled +- // path (correct, slower than MMA; a prefill-MMA dequant is a deferred perf follow-on, like buun's +- // k_turbo3_tcq_dequant_f16). Without this, a prefill ubatch would fall to the else-branch with no +- // cast and emit raw TCQ to FA with an UN-rotated Q → wrong logits. +- const bool is_tcq_kv = (k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ); // opencoti-hook: TCQ FA-VEC (#447/#500) — no materialize-lift, always fuse +- // opencoti-hook: asymmetric TCQ (#513). TCQ has no f16 materialize-lift, so the only correct path is +- // the in-register fused FA-VEC read. Allow K!=V when BOTH sides are TCQ (e.g. turbo3_tcq K / turbo2_tcq +- // V): K and V are FWHT-rotated identically, so the same Q-rotation + output inverse-WHT stay logit-exact +- // regardless of per-side trellis bit-width. Backed by the dedicated asymmetric FA-VEC instances +- // (fattn-vec-instance-turbo3_tcq-turbo2_tcq + reverse); the CUDA selector K!=V gate is relaxed to match. ++ // opencoti-hook #533: TCQ now HAS a materialize-lift (turbo-cpy.cu cpy_turbo{3,2}_tcq_f16: trellis ++ // codebook decode + inverse-FWHT → TRUE-space f16). So TCQ follows the SAME n_q threshold as plain ++ // turbo — fuse the in-register FA-VEC at decode (small n_q), de-fuse to the else-branch (ggml_cast ++ // TCQ→f16 → FA-MMA tensor cores) at prefill (large n_q). This recovers prefill throughput (the forced ++ // FA-VEC collapse, ~376→MMA-class) and lets TCQ compose with DCA's #444 all-KV f16-lift. The ++ // else-branch cast de-rotates K,V fully to TRUE space, so the UN-rotated Q there is logit-exact ++ // (Parseval). Previously (#447/#500) "|| is_tcq_kv" force-fused TCQ everywhere because no lift existed. ++ // opencoti-hook: asymmetric TCQ (#513). Allow K!=V when BOTH sides are TCQ (e.g. turbo3_tcq K / ++ // turbo2_tcq V). At DECODE this fuses the in-register FA-VEC read: K and V are FWHT-rotated identically, ++ // so the same Q-rotation + output inverse-WHT stay logit-exact regardless of per-side trellis bit-width ++ // (backed by the dedicated asymmetric FA-VEC instances fattn-vec-instance-turbo3_tcq-turbo2_tcq + ++ // reverse; the CUDA selector K!=V gate is relaxed to match). At PREFILL each side de-fuses and materializes ++ // independently to TRUE-space f16 via its own cpy_turbo{3,2}_tcq_f16 kernel (#533), then plain MMA. + const bool both_tcq = (k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) && + (v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ); + const bool fused_turbo = + cparams.flash_attn && kq_b == nullptr && + is_turbo_vec(k->type) && is_turbo_vec(v->type) && (k->type == v->type || both_tcq) && + (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) && +- (tbv_nq_per_strm <= tbv_fuse_max_nq || is_tcq_kv); ++ tbv_nq_per_strm <= tbv_fuse_max_nq; // opencoti #533: TCQ de-fuses at prefill (now has a materialize-lift) + + if (fused_turbo) { + // forward-rotate Q (×scale_inv → signs₁ → WHT → 1/√G·signs₂); group auto-picks 128 since +@@ -2121,7 +2129,8 @@ ggml_tensor * llm_graph_context::build_attn_mha( + q = ggml_turbo_wht(ctx0, q, /*direction=*/0, /*group_size=*/0, /*scale=*/nullptr); + } else { + const auto is_turbo = [](enum ggml_type t) { +- return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_TURBO8_0; ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_TURBO8_0 ++ || t == GGML_TYPE_TURBO3_TCQ || t == GGML_TYPE_TURBO2_TCQ; // opencoti #533: TCQ→f16 materialize-lift (prefill-MMA + DCA) + }; + if (is_turbo(k->type)) { k = ggml_cast(ctx0, k, GGML_TYPE_F16); } + if (is_turbo(v->type)) { v = ggml_cast(ctx0, v, GGML_TYPE_F16); } +@@ -2186,6 +2195,15 @@ ggml_tensor * llm_graph_context::build_attn_mha( + ggml_flash_attn_ext_add_sinks(cur, sinks); + ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32); + ++ // opencoti-hook: sparse-attn #551 — attach the Quest block bitmask (parked by build_attn's ++ // store block) as the FA op's src[7]. launch_fattn reads src[7] and passes it to the FA-VEC ++ // kernel for per-block skipping. Consume-once (clear) so it never leaks to the next FA op. ++ // src[5]/src[6] are reserved for the streaming/position-window k_tail/v_tail. ++ if (opencoti_sparse_block_sel) { ++ cur->src[7] = opencoti_sparse_block_sel; ++ opencoti_sparse_block_sel = nullptr; ++ } ++ + if (fused_turbo) { + // opencoti-hook: turboquant perf-lift (M6-S2 Level B). Inverse-WHT + InnerQ un-equalize + // on the SMALL FA output (signs₂ → WHT → 1/√G·signs₁ → ×scale_inv), recovering true-space +@@ -2767,6 +2785,100 @@ ggml_tensor * llm_graph_context::build_attn( + + ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); + ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); ++ ++ // opencoti-hook: sparse-attn — #551 B1/B2. Reduce the just-written cache K into the per-block ++ // kmin/kmax side-cache (fill), then run the Quest selector (Q·bounds upper bound → top-K + ++ // recent/sink → I32 block bitmask). The selector reads the fill RESULT so the scheduler orders ++ // select-after-fill; the bitmask is parked in opencoti_sparse_block_sel and attached as the ++ // standard FA op's src[7] in build_attn_mha. Off ⇒ get_kbounds nullptr ⇒ no op ⇒ byte-identical. ++ // opencoti-hook: sparse-attn-vslash — #551. mode==0 (default) = Quest bound (this block, unchanged); ++ // mode==1 = vertical-slash content-driven selector (the separate block immediately below). ++ if (cparams.sparse_attn_enabled && cparams.sparse_attn_mode == 0) { ++ ggml_tensor * kbounds = mctx_cur->get_kbounds(ctx0, il); ++ if (kbounds) { ++ ggml_tensor * k_full = mctx_cur->get_k(ctx0, il); ++ // opencoti #551: block size must be a multiple of the FA-VEC tile width (nthreads=128) so the ++ // compacted-list consumer's tiles align to blocks (sparse_m = B/128 ≥ 1). Clamp up. ++ const int B_raw = (int) cparams.sparse_attn_block_size; ++ const int B = B_raw < 128 ? 128 : (B_raw / 128) * 128; ++ // opencoti #551 sub-block Quest bound: kbounds is filled at the finer raw granularity (subB, ++ // = kbounds alloc granularity), and the score coalesces SUB sub-blocks per walk-block B via a ++ // max. Tightens the kmin/kmax envelope (B=128 is ~8x Quest's page → loose). subB must divide ++ // the walk B and be ≤ B; otherwise fall back to subB=B (SUB=1 ⇒ identical to plain B-block). ++ const int subB = (B_raw > 0 && B_raw <= B && B % B_raw == 0) ? B_raw : B; ++ const int SUB = B / subB; ++ // opencoti bug-744: the fill reads k_cur (the NEW tokens, pre-quantization f16/f32) instead ++ // of the quantized cache view, so the Quest kmin/kmax bound composes with EVERY cache KV ++ // type (q8_0/q4_0/turbo*/tcq*). k_full is kept for the n_kv/n_block sizing below. ++ ggml_tensor * k_fill = ggml_is_contiguous(k_cur) ? k_cur : ggml_cont(ctx0, k_cur); ++ ggml_tensor * fill = ggml_sparse_attn_fill(ctx0, kbounds, k_fill, k_idxs, subB); ++ ggml_build_forward_expand(gf, fill); ++ // opencoti #551 PERF: the FA-VEC DECODE kernel is the only consumer of block_sel; the ++ // MMA prefill kernel ignores it. Building the O(n_block·n_q) score+select at prefill ++ // (n_q>1) is pure overhead for zero gain (measured 5-8× prefill collapse). Run the ++ // selector ONLY at decode (n_q==1); fill still runs every step so kbounds stays current. ++ if (q_cur->ne[2] == 1) { ++ // opencoti #551 bug-743 FIX: get_k() returns [head_dim, n_head_kv, n_kv, 1] — the KV ++ // length is ne[2], NOT ne[1] (=n_head_kv). Reading ne[1] gave n_kv≈8 ⇒ n_block=1 ⇒ the ++ // selector saw one block ⇒ FA-VEC walked only the first ~128 keys of the cache (garbage ++ // past 128 tokens; invisible at short ctx where 1 block covers all). Use the position axis. ++ const int n_kv = (int) k_full->ne[2]; ++ const int n_block = (n_kv + B - 1) / B; ++ // topk==0 ⇒ select-all (inert B2 byte-identical anchor). --sparse-attn-recent is in ++ // TOKENS → round up to blocks; sink=1 forces block 0. select's src[1]=fill orders ++ // select-after-fill. topk==0xFFFFFFFF ⇒ auto: ¼ of blocks, floor 64. ++ const int recent_blk = ((int) cparams.sparse_attn_recent + B - 1) / B; ++ const int sink_blk = (int) cparams.sparse_attn_sink; ++ const int k_sel = (cparams.sparse_attn_topk == 0xFFFFFFFFu) ++ ? (n_block / 4 > 64 ? n_block / 4 : 64) ++ : (int) cparams.sparse_attn_topk; ++ // #551 PERF (amortized selection): when the KV cache exposes a persistent per-stream ++ // block_sel buffer (single-stream), the SELECT op writes into it in-place and re-selects ++ // only every cparams.sparse_attn_refresh decode tokens (in-kernel gate on the position ++ // from k_idxs), reusing the cached list in between. nullptr ⇒ transient per-step select. ++ ggml_tensor * block_sel_buf = mctx_cur->get_block_sel(ctx0, il); ++ opencoti_sparse_block_sel = ggml_sparse_attn_select( ++ ctx0, q_cur, fill, block_sel_buf, block_sel_buf ? k_idxs : nullptr, B, n_block, ++ k_sel, recent_blk, sink_blk, (int) cparams.sparse_attn_refresh, SUB); ++ ggml_build_forward_expand(gf, opencoti_sparse_block_sel); ++ } ++ } ++ } ++ // opencoti-hook: sparse-attn-vslash — #551 content-driven selector (mode==1), MInference prefill-window. ++ // The selector is estimated ONCE at PREFILL (n_q>1) over the last q_win prompt rows — which, for a ++ // retrieval prompt, ARE the question and genuinely attend the needle — by softmax block-mass (vertical ++ // importance), written into the persistent per-layer block_sel buffer. At DECODE (n_q==1) we do NOT ++ // recompute (the prompt queries are gone and a single decode query is unreliable at long ctx — the ++ // q_win=1 decode estimate failed the 40k gate AND cost a full attention pass per step, bug-741); we ++ // simply re-attach the cached block_sel as FA src[7]. The MMA prefill path is dense and ignores src[7], ++ // so the prefill emit only side-writes the buffer (opencoti_sparse_block_sel stays null at prefill). ++ if (cparams.sparse_attn_enabled && cparams.sparse_attn_mode == 1) { ++ ggml_tensor * k_full = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * block_sel_buf = k_full ? mctx_cur->get_block_sel(ctx0, il) : nullptr; ++ if (block_sel_buf) { ++ const int B_raw = (int) cparams.sparse_attn_block_size; ++ const int B = B_raw < 128 ? 128 : (B_raw / 128) * 128; ++ const int n_kv = (int) k_full->ne[2]; // position axis (bug-743) ++ const int n_block = (n_kv + B - 1) / B; ++ const int recent_blk = ((int) cparams.sparse_attn_recent + B - 1) / B; ++ const int sink_blk = (int) cparams.sparse_attn_sink; ++ const int k_sel = (cparams.sparse_attn_topk == 0xFFFFFFFFu) ++ ? (n_block / 4 > 64 ? n_block / 4 : 64) ++ : (int) cparams.sparse_attn_topk; ++ if (q_cur->ne[2] > 1) { ++ // PREFILL: estimate over the last q_win rows of this chunk; refresh=1 (always compute, never ++ // amortize at prefill). The final prompt chunk's write persists into decode. Side-write only. ++ const int q_win = q_cur->ne[2] < 8 ? (int) q_cur->ne[2] : 8; // #551 fast correctness read (was 64; prefill cost ∝ q_win) ++ ggml_tensor * sel = ggml_sparse_attn_vslash( ++ ctx0, q_cur, k_full, block_sel_buf, k_idxs, B, n_block, ++ k_sel, recent_blk, sink_blk, /*refresh=*/1, q_win); ++ ggml_build_forward_expand(gf, sel); ++ } else { ++ // DECODE: reuse the cached prefill selection — attach as FA src[7], no recompute. ++ opencoti_sparse_block_sel = block_sel_buf; ++ } ++ } ++ } + } + + const auto & kq_mask = inp->get_kq_mask(); +@@ -3070,6 +3182,82 @@ ggml_tensor * llm_graph_context::build_attn( + ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); + } + ++ // opencoti-hook: sparse-attn #551 — Quest block selector on the iSWA path, GLOBAL (!is_swa) ++ // layers ONLY. SWA layers are already windowed (block-selection is moot); the global ++ // full-attention layers are exactly the long-ctx KV-bandwidth hogs sparse targets. Mirrors the ++ // unified overload's store block (build_attn @ ~2794). Clear unconditionally first so a global ++ // layer's bitmask never bleeds into a following SWA layer's build_attn_mha consume (the ++ // window_layer branch below does not consume it). Engages only on build_attn_mha (resident KV); ++ // the POSITION_WINDOW overflow tactic falls through without sparse (first-cut limitation). ++ opencoti_sparse_block_sel = nullptr; ++ // opencoti-hook: sparse-attn-vslash — #551. mode==0 (default) = Quest bound (this block); mode==1 = ++ // vertical-slash content-driven selector (the separate block below). iSWA global (resident-KV) path. ++ if (!is_swa && cparams.sparse_attn_enabled && cparams.sparse_attn_mode == 0 && k_cur) { ++ ggml_tensor * kbounds = mctx_cur->get_kbounds(ctx0, il); ++ if (kbounds) { ++ ggml_tensor * k_full = mctx_cur->get_k(ctx0, il); ++ // opencoti #551: block size must be a multiple of the FA-VEC tile width (nthreads=128). ++ const int B_raw = (int) cparams.sparse_attn_block_size; ++ const int B = B_raw < 128 ? 128 : (B_raw / 128) * 128; ++ // opencoti #551 sub-block Quest bound (iSWA global path) — see unified path for rationale. ++ const int subB = (B_raw > 0 && B_raw <= B && B % B_raw == 0) ? B_raw : B; ++ const int SUB = B / subB; ++ // opencoti bug-744: fill from k_cur (new tokens, pre-quant) so sparse composes with all KV types ++ // (iSWA global path). k_full kept for n_kv/n_block sizing below. ++ ggml_tensor * k_fill = ggml_is_contiguous(k_cur) ? k_cur : ggml_cont(ctx0, k_cur); ++ ggml_tensor * fill = ggml_sparse_attn_fill(ctx0, kbounds, k_fill, inp->get_k_idxs(), subB); ++ ggml_build_forward_expand(gf, fill); ++ // opencoti #551 PERF: selector (score+select) only at decode (n_q==1) — MMA prefill ignores ++ // block_sel, so building it at prefill is pure overhead. Fill runs every step (kbounds current). ++ if (q_cur->ne[2] == 1) { ++ // opencoti #551 bug-743 FIX: KV length is k_full->ne[2] (position axis), NOT ne[1] ++ // (=n_head_kv). ne[1] gave n_block=1 ⇒ only the first ~128 keys attended. (iSWA global path.) ++ const int n_kv = (int) k_full->ne[2]; ++ const int n_block = (n_kv + B - 1) / B; ++ const int recent_blk = ((int) cparams.sparse_attn_recent + B - 1) / B; ++ const int sink_blk = (int) cparams.sparse_attn_sink; ++ const int k_sel = (cparams.sparse_attn_topk == 0xFFFFFFFFu) ++ ? (n_block / 4 > 64 ? n_block / 4 : 64) ++ : (int) cparams.sparse_attn_topk; ++ // #551 PERF (amortized): persistent per-stream block_sel buffer (single-stream) ⇒ in-place ++ // write + re-select every cparams.sparse_attn_refresh decode tokens; nullptr ⇒ transient. ++ ggml_tensor * block_sel_buf = mctx_cur->get_block_sel(ctx0, il); ++ opencoti_sparse_block_sel = ggml_sparse_attn_select( ++ ctx0, q_cur, fill, block_sel_buf, block_sel_buf ? inp->get_k_idxs() : nullptr, ++ B, n_block, k_sel, recent_blk, sink_blk, (int) cparams.sparse_attn_refresh, SUB); ++ ggml_build_forward_expand(gf, opencoti_sparse_block_sel); ++ } ++ } ++ } ++ // opencoti-hook: sparse-attn-vslash — #551 content-driven selector (mode==1), iSWA global path. ++ // MInference prefill-window: estimate ONCE at prefill (n_q>1) over the last q_win prompt rows by softmax ++ // block-mass into the persistent block_sel buffer; at decode reuse the cached selection (no recompute). ++ // Mirrors the unified-site placement above (see rationale + bug-741 there). ++ if (!is_swa && cparams.sparse_attn_enabled && cparams.sparse_attn_mode == 1 && k_cur) { ++ ggml_tensor * k_full = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * block_sel_buf = k_full ? mctx_cur->get_block_sel(ctx0, il) : nullptr; ++ if (block_sel_buf) { ++ const int B_raw = (int) cparams.sparse_attn_block_size; ++ const int B = B_raw < 128 ? 128 : (B_raw / 128) * 128; ++ const int n_kv = (int) k_full->ne[2]; // position axis (bug-743) ++ const int n_block = (n_kv + B - 1) / B; ++ const int recent_blk = ((int) cparams.sparse_attn_recent + B - 1) / B; ++ const int sink_blk = (int) cparams.sparse_attn_sink; ++ const int k_sel = (cparams.sparse_attn_topk == 0xFFFFFFFFu) ++ ? (n_block / 4 > 64 ? n_block / 4 : 64) ++ : (int) cparams.sparse_attn_topk; ++ if (q_cur->ne[2] > 1) { ++ const int q_win = q_cur->ne[2] < 8 ? (int) q_cur->ne[2] : 8; // #551 fast correctness read (was 64; prefill cost ∝ q_win) ++ ggml_tensor * sel = ggml_sparse_attn_vslash( ++ ctx0, q_cur, k_full, block_sel_buf, inp->get_k_idxs(), B, n_block, ++ k_sel, recent_blk, sink_blk, /*refresh=*/1, q_win); ++ ggml_build_forward_expand(gf, sel); ++ } else { ++ opencoti_sparse_block_sel = block_sel_buf; ++ } ++ } ++ } ++ + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + + ggml_tensor * q = q_cur; +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index 79fefd0..ba644cc 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -30,7 +30,8 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter, +- const layer_reuse_cb & reuse) : hparams(model.hparams), unified(unified) { ++ const layer_reuse_cb & reuse, ++ uint32_t sparse_attn_block_size) : hparams(model.hparams), unified(unified) { + + // chain filters + const layer_filter_cb filter_base = [&](int32_t il) { +@@ -69,7 +70,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + model, type_k, type_v, + v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, +- n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); ++ n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, sparse_attn_block_size); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + +@@ -77,7 +78,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + model, type_k, type_v, + v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, +- n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse); ++ n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, sparse_attn_block_size); + } + + void llama_kv_cache_iswa::clear(bool data) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 94feb62..d274b0a 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -45,7 +45,11 @@ public: + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter, +- const layer_reuse_cb & reuse); ++ const layer_reuse_cb & reuse, ++ // opencoti #551 sparse-attn — forwarded to both kv_base and ++ // kv_swa. Quest block-selector B_SEL; 0 = off (no side-cache). ++ // See docs/features/sparse_attn.md. ++ uint32_t sparse_attn_block_size = 0); + + ~llama_kv_cache_iswa() = default; + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 4bab0d1..3687725 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -332,12 +332,17 @@ llama_kv_cache::llama_kv_cache( + uint32_t n_swa, + llama_swa_type swa_type, + const layer_filter_cb & filter, +- const layer_reuse_cb & reuse) : ++ const layer_reuse_cb & reuse, ++ uint32_t sparse_attn_block_size) : + model(model), hparams(model.hparams), v_trans(v_trans), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type) { + + GGML_ASSERT(kv_size % n_pad == 0); + ++ // opencoti #551 sparse-attn — stash B_SEL (0 = off) for the side-cache alloc ++ // below + the selector (S2). See docs/features/sparse_attn.md. ++ sparse_attn_block_size_ = sparse_attn_block_size; ++ + // opencoti F4 M3 Phase 2 — see docs/decisions/0001-lazy-slot-context.md + // Initialize the hard cap and the soft cap. kv_size_initial==0 (the + // legacy / Phase-1 default) means "no soft cap" so the whole kv_size +@@ -424,6 +429,37 @@ llama_kv_cache::llama_kv_cache( + window_cells, kv_size, kv_size - window_cells); + } + ++ // opencoti F5 rolling-KV S0 — per-layer residency budget (task #585). ++ // OPENCOTI_KV_LOCAL_LAYERS="0,1,2,4,11" marks provably-local layers (from the HAL ++ // attention-locality profile — layers whose retrieval mass stays inside the recent ++ // window) as FULLY RESIDENT: window_cells==kv_size ⇒ no host tail for that layer. ++ // Full-attention models (Qwen) then stream a tail on only the RETRIEVING layers, ++ // the way Gemma iSWA already does for free. Manual env gate for the live niah gate; ++ // S4 auto-derives the set from an online prefill locality estimate. Empty/unset ⇒ ++ // uniform behavior (byte-inert). Only meaningful in window_mode. GATE per-layer niah, ++ // never widen past retrieval survival. See docs/features/rolling_kv_step_prefetch.md. ++ std::vector kv_local_layer(hparams.n_layer, false); ++ if (window_mode) { ++ const char * ll = getenv("OPENCOTI_KV_LOCAL_LAYERS"); ++ if (ll && *ll) { ++ std::string s(ll); ++ size_t pos = 0; ++ while (pos < s.size()) { ++ const size_t comma = s.find(',', pos); ++ const std::string tok = s.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); ++ if (!tok.empty()) { ++ const long v = strtol(tok.c_str(), nullptr, 10); ++ if (v >= 0 && (uint32_t) v < hparams.n_layer) kv_local_layer[v] = true; ++ } ++ if (comma == std::string::npos) break; ++ pos = comma + 1; ++ } ++ uint32_t nloc = 0; for (const bool b : kv_local_layer) nloc += b ? 1u : 0u; ++ LLAMA_LOG_INFO("%s: rolling-kv per-layer residency — %u/%u layer(s) forced RESIDENT " ++ "(no host tail) via OPENCOTI_KV_LOCAL_LAYERS\n", __func__, nloc, hparams.n_layer); ++ } ++ } ++ + // define a comparator for the buft -> ctx map to ensure that the order is well-defined: + struct ggml_backend_buft_comparator { + bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const { +@@ -457,8 +493,14 @@ llama_kv_cache::llama_kv_cache( + auto & ctx_map = ctx_map_per_stream[s]; + auto it = ctx_map.find(buft); + if (it == ctx_map.end()) { ++ // opencoti #551 sparse-attn — the per-(buft,stream) metadata pool budgets ++ // 4 tensors/layer (k, v, k_cpu, v_cpu worst case). When sparse-attn is on, ++ // each device layer also allocates kmin + kmax into this ctx, so bump the ++ // budget to 8/layer (covers k,v,kmin,kmax + the cpu-coresident worst case ++ // with margin). no_alloc ⇒ metadata only, so the KV data buffer is ++ // unaffected; off (block_size==0) keeps the original 4u (byte-identical). + ggml_init_params params = { +- /*.mem_size =*/ size_t(4u*n_layer_kv*ggml_tensor_overhead()), ++ /*.mem_size =*/ size_t((sparse_attn_block_size > 0 ? 8u : 4u)*n_layer_kv*ggml_tensor_overhead()), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; +@@ -607,8 +649,16 @@ llama_kv_cache::llama_kv_cache( + // host tail tensor spans [wc, kv_size) (tail_c cells); both carry ALL + // heads. Outside window mode wc == kv_size and tail_c == 0, so the + // tensor shapes below are byte-identical to the pre-3c layout. +- const uint32_t wc = window_mode ? window_cells : kv_size; +- const uint32_t tail_c = window_mode ? (kv_size - window_cells) : 0; ++ // opencoti F5 rolling-KV S0 (#585): a layer marked local (OPENCOTI_KV_LOCAL_LAYERS) ++ // is FULLY RESIDENT — wc==kv_size, no host tail — so only retrieving layers stream. ++ const bool layer_local = window_mode && kv_local_layer[il]; ++ // A local layer behaves EXACTLY like a non-window layer: fully resident ++ // (wc == kv_size), NO host tail, and window_cells sentinel 0 → the accessors ++ // and graph take the plain-FA resident path (never touch a 0-row tail). Gate ++ // ALL per-layer window/tail machinery on layer_window, not the global window_mode. ++ const bool layer_window = window_mode && !layer_local; ++ const uint32_t wc = layer_window ? window_cells : kv_size; ++ const uint32_t tail_c = layer_window ? (kv_size - window_cells) : 0; + + // opencoti F4 M3 Phase 4 — per-stream tensor split. + // Each stream gets its own OWNING 3-D tensor of shape +@@ -624,6 +674,12 @@ llama_kv_cache::llama_kv_cache( + std::vector v_cpu_per_stream; + std::vector k_stream; + std::vector v_stream; ++ // opencoti #551 sparse-attn — per-stream kbounds side-cache. Stay EMPTY ++ // when sparse-attn off (sparse_attn_block_size == 0) ⇒ byte-identical. ++ std::vector kbounds_per_stream; ++ // opencoti #551 sparse-attn PERF — per-stream persistent selected-block list (amortized ++ // selection). EMPTY when sparse off OR n_stream > 1 (per-step transient fallback). ++ std::vector block_sel_per_stream; + + for (uint32_t s = 0; s < n_stream; ++s) { + ggml_context * ctx_s = ctx_for_buft(buft, s); +@@ -673,15 +729,15 @@ llama_kv_cache::llama_kv_cache( + // mode forces gpu_heads == 0), so exactly one branch sizes the + // tensor. The host buffer is the pinned cpu_buft when CUDA is the + // layer dev — full PCIe DMA for the stream-back. +- if (gpu_heads > 0 || window_mode) { ++ if (gpu_heads > 0 || layer_window) { + ggml_context * ctx_cpu_s = ctx_for_buft(cpu_buft, s); + if (!ctx_cpu_s) { + throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); + } + // window mode: ALL heads, tail_c cells; M2: CPU head subset, kv_size cells. +- const uint32_t host_k_embd = window_mode ? n_embd_k_gqa : n_embd_k_cpu; +- const uint32_t host_v_embd = window_mode ? n_embd_v_gqa : n_embd_v_cpu; +- const uint32_t host_cells = window_mode ? tail_c : kv_size; ++ const uint32_t host_k_embd = layer_window ? n_embd_k_gqa : n_embd_k_cpu; ++ const uint32_t host_v_embd = layer_window ? n_embd_v_gqa : n_embd_v_cpu; ++ const uint32_t host_cells = layer_window ? tail_c : kv_size; + ggml_tensor * k_cpu_s = has_k + ? ggml_new_tensor_3d(ctx_cpu_s, type_k, host_k_embd, host_cells, 1) + : nullptr; +@@ -690,8 +746,8 @@ llama_kv_cache::llama_kv_cache( + : nullptr; + // Name distinguishes the two layouts for state-IO grep: + // "cache_k_tail" (position tail) vs "cache_k_cpu" (head half). +- const char * k_base = window_mode ? "cache_k_tail" : "cache_k_cpu"; +- const char * v_base = window_mode ? "cache_v_tail" : "cache_v_cpu"; ++ const char * k_base = layer_window ? "cache_k_tail" : "cache_k_cpu"; ++ const char * v_base = layer_window ? "cache_v_tail" : "cache_v_cpu"; + if (k_cpu_s) { + if (n_stream == 1) { + ggml_format_name(k_cpu_s, "%s_l%d", k_base, il); +@@ -731,6 +787,33 @@ llama_kv_cache::llama_kv_cache( + v_stream.push_back(v_s + ? ggml_view_2d(ctx_s, v_s, n_embd_v_gpu, wc, v_s->nb[1], 0) + : nullptr); ++ ++ // opencoti #551 sparse-attn — per-(KV-head, block) min/max bounds, ++ // allocated ONLY when enabled. n_block_sel = ceil(wc / B_SEL). Lives in ++ // the device ctx_s alongside k_s (same n_embd_k_gpu dim0 as the resident ++ // window), f16 regardless of KV-quant (bounds come from dequant K at the ++ // S2 write). Off ⇒ vectors stay empty ⇒ no tensor, byte-identical. ++ if (sparse_attn_block_size > 0 && k_s) { ++ const uint32_t n_block_sel = (wc + sparse_attn_block_size - 1) / sparse_attn_block_size; ++ // [n_embd_k_gpu, n_block_sel, 2] f16 : [..,0]=min, [..,1]=max ++ ggml_tensor * kbounds_s = ggml_new_tensor_3d(ctx_s, GGML_TYPE_F16, n_embd_k_gpu, n_block_sel, 2); ++ if (n_stream == 1) { ++ ggml_format_name(kbounds_s, "cache_kbounds_l%d", il); ++ } else { ++ ggml_format_name(kbounds_s, "cache_kbounds_l%d_s%u", il, s); ++ } ++ kbounds_per_stream.push_back(kbounds_s); ++ ++ // opencoti #551 sparse-attn PERF (amortized selection) — persistent selected-block ++ // list, single-stream only (n_stream > 1 falls back to per-step transient selection). ++ // [n_block_sel + 1, 1] I32: [0,n_block_sel) = sorted −1-padded compacted list, ++ // [n_block_sel] = reserved scratch. Init to [0,−1,…] post-alloc (see below). ++ if (n_stream == 1) { ++ ggml_tensor * block_sel_s = ggml_new_tensor_2d(ctx_s, GGML_TYPE_I32, n_block_sel + 1, 1); ++ ggml_format_name(block_sel_s, "cache_block_sel_l%d", il); ++ block_sel_per_stream.push_back(block_sel_s); ++ } ++ } + } + + map_layer_ids[il] = layers.size(); +@@ -741,7 +824,9 @@ llama_kv_cache::llama_kv_cache( + k_stream, v_stream, + k_cpu_per_stream, v_cpu_per_stream, + gpu_heads, +- window_mode ? window_cells : 0u, ++ layer_window ? window_cells : 0u, ++ kbounds_per_stream, ++ block_sel_per_stream, + }); + } + +@@ -826,6 +911,23 @@ llama_kv_cache::llama_kv_cache( + } + } + ++ // opencoti #551 sparse-attn PERF — initialize each persistent block_sel list to [0, −1, −1, …] ++ // (sink block 0 selected, rest empty). ensure_cleared() only zeroes the K/V tensors, never ++ // block_sel, so without this the buffer is uninitialized garbage on the first decode steps before ++ // the selector's first refresh. [0,−1,…] makes a never-refreshed buffer attend ONLY the sink ++ // block — safe (the FA-VEC consumer breaks at the first −1; never empty ⇒ no NaN, never OOB). ++ if (!model.hparams.no_alloc) { ++ for (const auto & layer : layers) { ++ for (ggml_tensor * bs : layer.block_sel_per_stream) { ++ if (!bs) { continue; } ++ const int64_t n = bs->ne[0]; ++ std::vector init((size_t) n, -1); ++ init[0] = 0; ++ ggml_backend_tensor_set(bs, init.data(), 0, (size_t) n * sizeof(int32_t)); ++ } ++ } ++ } ++ + if (model.hparams.no_alloc) { + // opencoti F4 M3 Phase 1+2 — dummy buffers are size 0; treat all + // cells as "already cleared" so ensure_cleared is a no-op. +@@ -2831,6 +2933,34 @@ ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t + return result; + } + ++// opencoti-hook: sparse-attn — #551. Returns the per-(layer,stream) kbounds side-cache ++// tensor [n_embd_k_gpu, n_block_sel, 2] f16, or nullptr when sparse-attn is off (the ++// side-cache was never allocated). Referenced raw as a graph src (like k_l); the fill op ++// views it in-place. n_kv is irrelevant — kbounds spans all selectable blocks. ++ggml_tensor * llama_kv_cache::get_kbounds(ggml_context * ctx, int32_t il, const slot_info & sinfo) const { ++ GGML_UNUSED(ctx); ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ if (layer.kbounds_per_stream.empty()) { ++ return nullptr; ++ } ++ return layer.kbounds_per_stream[sinfo.s0]; ++} ++ ++// opencoti-hook: sparse-attn — #551 PERF. Returns the per-(layer,stream) persistent selected-block ++// list tensor [n_block_sel + 1, 1] I32, or nullptr when sparse-attn is off OR multi-stream (the ++// vector is only populated for n_stream == 1). The SELECT op views it in-place (cpy idiom) and ++// re-writes it only every cparams.sparse_attn_refresh decode tokens; FA-VEC reads it every step. ++ggml_tensor * llama_kv_cache::get_block_sel(ggml_context * ctx, int32_t il, const slot_info & sinfo) const { ++ GGML_UNUSED(ctx); ++ const int32_t ikv = map_layer_ids.at(il); ++ const auto & layer = layers[ikv]; ++ if (layer.block_sel_per_stream.empty()) { ++ return nullptr; ++ } ++ return layer.block_sel_per_stream[sinfo.s0]; ++} ++ + ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = map_layer_ids.at(il); + const auto & layer = layers[ikv]; +@@ -5146,6 +5276,16 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons + return kv->get_v(ctx, il, n_kv, sinfos[i_cur]); + } + ++// opencoti-hook: sparse-attn — #551. Per-batch wrapper around the kbounds accessor. ++ggml_tensor * llama_kv_cache_context::get_kbounds(ggml_context * ctx, int32_t il) const { ++ return kv->get_kbounds(ctx, il, sinfos[i_cur]); ++} ++ ++// opencoti-hook: sparse-attn — #551 PERF. Per-batch wrapper around the persistent block_sel accessor. ++ggml_tensor * llama_kv_cache_context::get_block_sel(ggml_context * ctx, int32_t il) const { ++ return kv->get_block_sel(ctx, il, sinfos[i_cur]); ++} ++ + // opencoti F5 M3 neo — per-batch wrappers around the head-split accessors. + ggml_tensor * llama_kv_cache_context::get_k_gpu(ggml_context * ctx, int32_t il) const { + return kv->get_k_gpu(ctx, il, n_kv, sinfos[i_cur]); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +index 5e8ea9b..c6027f1 100644 +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -136,7 +136,11 @@ public: + uint32_t n_swa, + llama_swa_type swa_type, + const layer_filter_cb & filter, +- const layer_reuse_cb & reuse); ++ const layer_reuse_cb & reuse, ++ // opencoti #551 sparse-attn — Quest block-selector KV-block ++ // size B_SEL (docs/features/sparse_attn.md). 0 = sparse-attn ++ // OFF: no kmin/kmax side-cache allocated (byte-identical). ++ uint32_t sparse_attn_block_size = 0); + + ~llama_kv_cache() = default; + +@@ -251,6 +255,11 @@ public: + ggml_tensor * get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; + ggml_tensor * get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; + ggml_tensor * get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ++ // opencoti-hook: sparse-attn — #551. Per-(layer,stream) kbounds side-cache, or nullptr when off. ++ ggml_tensor * get_kbounds(ggml_context * ctx, int32_t il, const slot_info & sinfo) const; ++ // opencoti-hook: sparse-attn — #551 PERF. Per-(layer,stream) persistent selected-block list, or ++ // nullptr when off / multi-stream (then the selector emits a transient per-step list instead). ++ ggml_tensor * get_block_sel(ggml_context * ctx, int32_t il, const slot_info & sinfo) const; + bool headinfer_split_active(int32_t il) const; + uint32_t headinfer_gpu_heads (int32_t il) const; + +@@ -423,6 +432,27 @@ private: + // [window_cells, kv_size). 0 ⇒ no position window (vanilla / M2 layout). + uint32_t window_cells = 0; + ++ // opencoti #551 sparse-attn — per-(KV-head·channel, block) min/max key ++ // bounds side-cache for the Quest block-selector. EMPTY when sparse-attn ++ // is off (sparse_attn_block_size_ == 0) ⇒ no tensor allocated, ++ // byte-identical. Shape per stream: [n_embd_k_gpu, n_block_sel, 2] f16 ++ // ([..,0]=min, [..,1]=max), with n_block_sel = ceil(window_cells / B_SEL). ++ // Filled by GGML_OP_SPARSE_ATTN_FILL right after the cpy_k write (B1); ++ // read by the selector (B2) + FA-VEC block-skip (B2/S3). ++ // See docs/features/sparse_attn.md. ++ std::vector kbounds_per_stream; ++ ++ // opencoti #551 sparse-attn PERF (amortized selection) — per-stream persistent ++ // I32 selected-block list [n_block_sel + 1, 1]: slots [0,n_block_sel) hold the ++ // SORTED −1-padded compacted list (FA-VEC consumer), slot [n_block_sel] is a ++ // scratch pad reserved for future markers. Persists across decode steps so the ++ // GGML_OP_SPARSE_ATTN_SELECT kernel can re-run only every cparams.sparse_attn_refresh ++ // tokens (reading the decode position from k_idxs) and reuse the cached selection in ++ // between — amortizing the per-step selector cost. EMPTY when sparse off OR n_stream>1 ++ // (multi-stream falls back to per-step transient selection). Init to [0,−1,−1,…] so a ++ // never-refreshed buffer attends only the sink block (safe, never OOB / never empty). ++ std::vector block_sel_per_stream; ++ + // Convenience accessors for unified-mode (n_stream == 1) call + // sites. Returns nullptr in non-unified mode — that's the + // signal that the caller must use per-stream access instead. +@@ -471,6 +501,10 @@ private: + // this is the SWA type of the cache - not to be confused with the model SWA type + const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; + ++ // opencoti #551 sparse-attn — Quest block-selector KV-block size B_SEL. ++ // 0 = sparse-attn off (no kmin/kmax side-cache). Set from the ctor arg. ++ uint32_t sparse_attn_block_size_ = 0; ++ + // ggml contexts for the KV cache along with the allocated backend buffers: + std::vector> ctxs_bufs; + +@@ -622,6 +656,11 @@ public: + ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; + ggml_tensor * get_v(ggml_context * ctx, int32_t il) const; + ++ // opencoti-hook: sparse-attn — #551. Per-(layer,stream) kbounds side-cache, or nullptr when off. ++ ggml_tensor * get_kbounds(ggml_context * ctx, int32_t il) const; ++ // opencoti-hook: sparse-attn — #551 PERF. Per-batch wrapper around the persistent block_sel accessor. ++ ggml_tensor * get_block_sel(ggml_context * ctx, int32_t il) const; ++ + // opencoti F5 M3 neo — head-split-aware K/V accessors. Per-batch wrappers + // that delegate to the underlying llama_kv_cache with the current n_kv / + // slot info. See llama-kv-cache.h for semantics. +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +index b18a498..017b755 100644 +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2104,7 +2104,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + cparams.n_ubatch, + 1, + filter, +- reuse); ++ reuse, ++ // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) ++ cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); + } else { + GGML_ASSERT(!hparams.is_swa_any()); + +@@ -2133,7 +2135,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + hparams.n_swa, + hparams.swa_type, + filter, +- nullptr); ++ nullptr, ++ // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) ++ cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); + } + } + } diff --git a/patches/0090-rolling-kv-compute-reserve.patch b/patches/0090-rolling-kv-compute-reserve.patch new file mode 100644 index 0000000000000000000000000000000000000000..940572708570aabd26e99e3d5549b01a1568100b --- /dev/null +++ b/patches/0090-rolling-kv-compute-reserve.patch @@ -0,0 +1,333 @@ +From: opencoti +Subject: [PATCH 0089] rolling-KV two-pass compute-buffer reserve (bug-1342) + +The POSITION_WINDOW resident-window sizer (opencoti_compute_resident_window_cells) +held back a FIXED OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES = 1536 MiB for the compute +scratch that graph_reserve allocates AFTER the KV cache. But that compute buffer +SCALES with context — measured 3.63 GiB (3 800 045 568 B) at 262 144-524 288 ctx on +an RTX PRO 6000 — so the sizer kept ~2.1 GiB too much KV resident and graph_reserve +OOM'd at boot: + + ggml_gallocr_reserve_n_impl: failed to allocate CUDA0 buffer of size 3800045568 + graph_reserve: failed to allocate compute buffers + +Fix: measure the real compute buffer instead of guessing it. Two-pass context init, +gated to the GPU flash-attn residency-managed path so the common resident case stays +single-pass / byte-identical: + + Pass 1 (measure): create_memory with kv_window_measure_pass=true -> the window sizer + returns a MINIMAL resident window (tiny device KV, boots for ANY compute size). + sched_reserve() runs the pp worst-case reserve; sum the GPU backends' compute + buffer via backend_buf_exp_size. + Pass 2 (real): re-create_memory with kv_compute_reserve_mib = measured + 256 MiB + safety; the sizer now holds back the true compute buffer and sizes the window + against (budget - model - measured). sched_need_reserve=true; sched_reserve() again. + +The compute buffer is independent of the window/tail split (it depends on +n_ubatch x n_kv x n_embd, and n_kv = full ctx regardless of the resident split), so +one measurement suffices. kv_compute_reserve_mib / kv_window_measure_pass are INTERNAL +cparams (set by the context two-pass, not from llama_context_params); hybrid-memory +paths pass 0/false (two-pass disabled). Host-only change — see +docs/features/rolling_kv_compute_reserve.md. + +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -35,6 +35,16 @@ + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. + // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head, 2 = window. + uint32_t kv_residency_mode; ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) — ++ // see docs/features/rolling_kv_compute_reserve.md. INTERNAL (not user params): ++ // set by the llama_context two-pass measurement, NOT from llama_context_params. ++ // kv_compute_reserve_mib = MEASURED GPU compute-buffer size to hold back from the ++ // resident-window budget (0 = use the fixed OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ // default, i.e. pass 1 / two-pass disabled). kv_window_measure_pass forces the ++ // window sizer to a minimal resident window (tiny device KV, boots for ANY compute ++ // size) so pass 1 can measure the real compute buffer without OOM. ++ uint32_t kv_compute_reserve_mib; ++ bool kv_window_measure_pass; + // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention. + // 0 = off (M2 concat path runs; binary equivalent to upstream when + // headinfer_gpu_heads_frac == 1.0). +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -106,6 +106,12 @@ + cparams.pcie_bw_gbps = params.pcie_bw_gbps; + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md + cparams.kv_residency_mode = params.kv_residency_mode; ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). INTERNAL: ++ // the memory init below runs a measurement pass, then re-creates with the measured ++ // reserve; these start at the "pass 1 / default reserve" values. See ++ // docs/features/rolling_kv_compute_reserve.md. ++ cparams.kv_compute_reserve_mib = 0; ++ cparams.kv_window_measure_pass = false; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + cparams.neo_pipeline_mode = params.neo_pipeline_mode; + // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md +@@ -373,6 +379,18 @@ + } + } + ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) — ++ // see docs/features/rolling_kv_compute_reserve.md. The resident-window sizer must ++ // hold back the compute buffer (allocated AFTER the KV by sched_reserve), but its ++ // size scales with context and is only known post-reserve. So: PASS 1 creates a ++ // MINIMAL resident window (tiny device KV, boots for any compute size), sched_reserve ++ // measures the real GPU compute buffer, then PASS 2 re-creates the KV sized against ++ // (free - model - measured). Scoped to the GPU flash-attn residency-managed path so ++ // the common resident case is unaffected (the pass-2 result is identical there). ++ const bool opencoti_two_pass = !hparams.vocab_only ++ && cparams.flash_attn && cparams.offload_kqv ++ && !model.devices.empty() && cparams.kv_residency_mode != 1; ++ + // init the memory module + if (!hparams.vocab_only) { + llama_memory_params params_mem = { +@@ -382,6 +400,8 @@ + /*.ctx_type= */ cparams.ctx_type, + }; + ++ // opencoti bug-1342: pass 1 forces a minimal resident window (measure mode). ++ cparams.kv_window_measure_pass = opencoti_two_pass; + memory.reset(model.create_memory(params_mem, cparams)); + } + +@@ -450,6 +470,41 @@ + + sched_reserve(); + ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). Pass 1 ++ // (above) sized a minimal resident window so sched_reserve could not OOM; the real ++ // GPU compute buffer is now in backend_buf_exp_size. Sum it over the GPU backends, ++ // re-create the KV with that measured reserve, and re-reserve. The compute buffer is ++ // independent of the window/tail split, so the measurement is exact for pass 2. See ++ // docs/features/rolling_kv_compute_reserve.md. ++ if (opencoti_two_pass && !model.hparams.no_alloc) { ++ size_t gpu_compute = 0; ++ for (size_t i = 0; i < backend_ptrs.size(); ++i) { ++ if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) == GGML_BACKEND_DEVICE_TYPE_GPU) { ++ gpu_compute += backend_buf_exp_size[i]; ++ } ++ } ++ // + 256 MiB safety margin over the measured worst-case pp compute buffer. ++ const uint32_t measured_mib = (uint32_t) (gpu_compute / (1024 * 1024)) + 256; ++ LLAMA_LOG_INFO("%s: rolling-kv two-pass — measured GPU compute buffer = %u MiB; " ++ "re-sizing resident window against it\n", __func__, measured_mib); ++ cparams.kv_compute_reserve_mib = measured_mib; ++ cparams.kv_window_measure_pass = false; ++ ++ // free the pass-1 KV (incl. its host tail) BEFORE creating pass 2 to avoid a ++ // transient double host-tail allocation. ++ memory.reset(); ++ llama_memory_params params_mem2 = { ++ /*.type_k =*/ params.type_k, ++ /*.type_v =*/ params.type_v, ++ /*.swa_full =*/ params.swa_full, ++ /*.ctx_type= */ cparams.ctx_type, ++ }; ++ memory.reset(model.create_memory(params_mem2, cparams)); ++ ++ sched_need_reserve = true; ++ sched_reserve(); ++ } ++ + if (!cparams.flash_attn) { + if (ggml_is_quantized(params.type_v)) { + throw std::runtime_error("quantized V cache was requested, but this requires Flash Attention"); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -131,6 +131,15 @@ + // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), + // 1 = head (force M2 split), 2 = window (force POSITION_WINDOW). + uint32_t kv_residency_mode, ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve ++ // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md. ++ // compute_reserve_mib: MEASURED GPU compute-buffer size to hold ++ // back from the resident-window budget (0 = fixed default). ++ // window_measure_pass: force a MINIMAL resident window (tiny ++ // device KV) so the context two-pass can measure the real ++ // compute buffer without OOM. ++ uint32_t compute_reserve_mib, ++ bool window_measure_pass, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -138,6 +138,7 @@ + uint32_t kv_size, + uint32_t n_stream, + uint32_t vram_target_mib, ++ size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back + const std::function & filter) { + // No GPU offload → nothing is "GPU-resident" and nothing to spill; the M2 + // split is only meaningful for offloaded layers. Stay at vanilla. +@@ -187,9 +188,10 @@ + if (vram_target_mib > 0) { + budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); + } +- budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES +- ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES +- : 0; ++ // opencoti bug-1342: hold back the MEASURED compute buffer (two-pass) instead of the ++ // fixed 1536 MiB guess when the context supplied it; 0 → fall back to the default. ++ const size_t reserve = compute_reserve_bytes ? compute_reserve_bytes : OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES; ++ budget = budget > reserve ? budget - reserve : 0; + + if (kv_demand <= budget) { + LLAMA_LOG_INFO("%s: auto residency = GPU_RESIDENT (KV %.0f MiB fits budget " +@@ -235,10 +237,21 @@ + uint32_t n_stream, + uint32_t vram_target_mib, + uint32_t n_pad, ++ size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back ++ bool measure_pass, // opencoti bug-1342: pass 1 → minimal window (tiny device KV) + const std::function & filter) { + if (!offload || kv_size == 0) { + return kv_size; // no position-axis spill without GPU offload + } ++ // opencoti bug-1342 two-pass: the MEASUREMENT pass forces a MINIMAL resident window ++ // (one FA stride, ~0.5 MiB device KV) so pass 1 boots for ANY compute-buffer size. ++ // The context then measures the real compute buffer and re-sizes in pass 2. Aligned ++ // to lcm(n_pad, FATTN_KQ_STRIDE=256) so the tiny window is a valid FA tiling. ++ if (measure_pass) { ++ const uint32_t pad0 = n_pad ? n_pad : 32; ++ const uint32_t walign0 = pad0 >= 256 ? pad0 : 256; ++ return kv_size > walign0 ? walign0 : kv_size; ++ } + + size_t per_cell = 0; // bytes for ONE position cell across all this cache's KV layers + ggml_backend_dev_t dev = nullptr; +@@ -327,6 +340,8 @@ + uint32_t vram_target_mib, + float pcie_bw_gbps, + uint32_t kv_residency_mode, ++ uint32_t compute_reserve_mib, ++ bool window_measure_pass, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, +@@ -383,9 +398,14 @@ + // A non-negative frac is the manual escape hatch / upstream default, used + // verbatim. Every split decision below reads eff_gpu_heads_frac, not the raw + // ctor argument, so the sentinel never leaks into the allocation math. ++ // opencoti bug-1342: the compute-buffer reserve to hold back from the resident budget. ++ // compute_reserve_mib > 0 = the MEASURED GPU compute buffer (context two-pass, pass 2); ++ // 0 = pass 1 / two-pass disabled → the sizers fall back to the fixed default. ++ const size_t opencoti_compute_reserve_bytes = (size_t) compute_reserve_mib * 1024 * 1024; + const float eff_gpu_heads_frac = headinfer_gpu_heads_frac < 0.0f + ? opencoti_compute_auto_gpu_heads_frac(model, hparams, type_k, type_v, +- v_trans, offload, kv_size, n_stream, vram_target_mib, filter) ++ v_trans, offload, kv_size, n_stream, vram_target_mib, ++ opencoti_compute_reserve_bytes, filter) + : headinfer_gpu_heads_frac; + + // opencoti F5 M7 Stage 3c — position-axis window sizing. Computed + logged now so the +@@ -395,7 +415,7 @@ + // value is intentionally unused downstream at this sub-stage. See docs/features/advanced_kv.md. + const uint32_t window_cells = opencoti_compute_resident_window_cells( + model, hparams, type_k, type_v, v_trans, offload, kv_size, n_stream, +- vram_target_mib, n_pad, filter); ++ vram_target_mib, n_pad, opencoti_compute_reserve_bytes, window_measure_pass, filter); + // opencoti F5 M7 Stage 4 (#338) — position-window mode is driven by the + // --kv-residency-mode knob (cparams.kv_residency_mode): 0=auto, 1=head, 2=window. + // The ELIGIBILITY predicate is the validated POSITION_WINDOW class: the sizer found a +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -41,6 +41,12 @@ + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob; + // forwarded to both kv_base and kv_swa. 0 = auto, 1 = head, 2 = window. + uint32_t kv_residency_mode, ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve ++ // (bug-1342); forwarded to both kv_base and kv_swa (only the ++ // overflowing base cache windows). See ++ // docs/features/rolling_kv_compute_reserve.md. ++ uint32_t compute_reserve_mib, ++ bool window_measure_pass, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -26,6 +26,8 @@ + uint32_t vram_target_mib, + float pcie_bw_gbps, + uint32_t kv_residency_mode, ++ uint32_t compute_reserve_mib, ++ bool window_measure_pass, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, +@@ -70,6 +72,7 @@ + model, type_k, type_v, + v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, ++ compute_reserve_mib, window_measure_pass, + n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, sparse_attn_block_size); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); +@@ -78,6 +81,7 @@ + model, type_k, type_v, + v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, ++ compute_reserve_mib, window_measure_pass, + n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, sparse_attn_block_size); + } + +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2100,6 +2100,10 @@ + cparams.pcie_bw_gbps, + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob + cparams.kv_residency_mode, ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve ++ // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md ++ cparams.kv_compute_reserve_mib, ++ cparams.kv_window_measure_pass, + cparams.n_seq_max, + cparams.n_ubatch, + 1, +@@ -2130,6 +2134,10 @@ + cparams.pcie_bw_gbps, + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob + cparams.kv_residency_mode, ++ // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve ++ // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md ++ cparams.kv_compute_reserve_mib, ++ cparams.kv_window_measure_pass, + cparams.n_seq_max, + 1, + hparams.n_swa, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -55,6 +55,9 @@ + 0.0f, + // opencoti F5 M7 Stage 4 — hybrid memory does not thread the residency-mode knob; 0 = auto. + 0, ++ // opencoti F5 M7 bug-1342 — hybrid memory does not use the window two-pass; 0 / false. ++ 0, ++ false, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -56,6 +56,9 @@ + 0.0f, + // opencoti F5 M7 Stage 4 — hybrid-iswa does not thread the residency-mode knob; 0 = auto. + 0, ++ // opencoti F5 M7 bug-1342 — hybrid-iswa does not use the window two-pass; 0 / false. ++ 0, ++ false, + n_seq_max, + n_ubatch, + n_pad, diff --git a/patches/0091-multi-session-graph-nodes.patch b/patches/0091-multi-session-graph-nodes.patch new file mode 100644 index 0000000000000000000000000000000000000000..966d6afab49b9a5bcc9bedde31157b847e88799d --- /dev/null +++ b/patches/0091-multi-session-graph-nodes.patch @@ -0,0 +1,50 @@ +From: opencoti +Subject: [PATCH 0090] multi-session graph-node budget for --parallel >=2 (bug-2094) + +graph_max_nodes() sized the compute-graph meta pool as a single-stream budget +(8*n_tensors for dense archs; max(n_tokens*40, 32*n_tensors) for the qwen3next/ +qwen35/kimi linear-attn branch), INDEPENDENT of the stream count. But with a +non-unified KV cache (the default; --parallel N => kv_unified=false) the graph +builds per-stream KV views/masks/copies (the `for s < n_stream` loops throughout +llama-kv-cache.cpp), so the actual node count scales with the number of streams. +On a 48-layer model (Qwen2.5-14B-Instruct-1M) --parallel 2 therefore overflowed +the pool at graph build: + + llama.cpp/ggml/src/ggml.c:1790: not enough space in the context's memory pool + +(server aborts at boot; --parallel 1 was fine; independent of --kv-residency-mode +and of ctx). Fix: scale the node budget by n_stream = (kv_unified ? 1 : n_seq_max), +applied to both branches. Tied to the exact per-stream-KV-graph mechanism; a +UNIFIED cache keeps n_stream=1 -> byte-identical to upstream, so the factor +self-disables when there is no per-stream graph. + +Measured actual n_nodes (RTX 3090, ctx 8192): dense Qwen3-8B 1267/2131/3571 at +--parallel 1/2/4 (~776 nodes/stream); iSWA Gemma-4-A4B 2647/3367/4567 (~660/stream); +qwen35 Qwen3.5-9B 1856/2048/2368 (~170/stream). All boot with 2.5-4x headroom. +Verified on the original repro (bs2 RTX PRO 6000, 14B-1M-Q8_0 --parallel 2): CRASH +-> BOOT. Host-only (build:llamafile:make); no CUDA DSO, no restamp. + +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -2585,10 +2585,19 @@ + // + + uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { ++ // opencoti-hook: bug-2094 multi-session graph-node budget — see docs/features/rolling_kv_compute_reserve.md ++ // With a non-unified KV cache (--parallel N ⇒ kv_unified=false) the graph builds ++ // per-stream KV views/masks/copies (the `for s < n_stream` loops throughout ++ // llama-kv-cache.cpp), so the node count scales with the stream count. The upstream ++ // budgets below are single-stream; without this factor --parallel ≥2 overflows the ++ // graph-meta pool (ggml.c:1790 "not enough space in the context's memory pool"). A ++ // UNIFIED cache keeps n_stream=1 → byte-identical to upstream, so the factor ++ // self-disables exactly when there is no per-stream graph. ++ const uint32_t n_stream = cparams.kv_unified ? 1u : std::max(1u, cparams.n_seq_max); + if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) { +- return std::max(n_tokens * 40, 32u * model.n_tensors()); ++ return std::max(n_tokens * 40, 32u * model.n_tensors() * n_stream); + } +- uint32_t res = std::max(1024u, 8u*model.n_tensors()); ++ uint32_t res = std::max(1024u, 8u*model.n_tensors()) * n_stream; + for (const auto & lora : model.loras) { + res += lora->get_n_nodes(); + } diff --git a/patches/0092-cuda-build-ccache.patch b/patches/0092-cuda-build-ccache.patch new file mode 100644 index 0000000000000000000000000000000000000000..03adbaabc79a9d19214d442630f558e65035ec8a --- /dev/null +++ b/patches/0092-cuda-build-ccache.patch @@ -0,0 +1,71 @@ +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +index 4781516..5b1c26c 100644 +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -238,8 +238,15 @@ compile_gpu_sources_parallel() { + obj="$build_dir/${base}.o" + fi + +- # Skip if object file is newer than source +- if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then ++ # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness. The CUDA kernels ++ # live in .cuh HEADERS; template-instance .cu stubs never change, so comparing the object ++ # only to its .cu left STALE objects on header edits → stale DSO. Also require the object ++ # to be newer than the newest ggml-tree header ($OPENCOTI_CUDA_DEPS_NEWEST, exported by ++ # cuda.sh). Compilation runs through ccache (see cuda.sh), so a header edit then recompiles ++ # only the TUs whose expanded includes actually changed. The `-z` guard preserves the old ++ # .cu-only behaviour for any caller that doesn't export the var. ++ if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \ ++ && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then + echo "[$count/$total] Skipping: $base.cu (up to date)" + continue + fi +@@ -298,8 +305,11 @@ compile_ggml_core() { + local name="${base%.*}" + local obj="$build_dir/ggml-core-${name}.o" + +- # Skip if object file is newer than source +- if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then ++ # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness (see cuda.sh). Skip ++ # only when the object is newer than the source AND every ggml-tree header, so a header ++ # edit correctly forces a recompile. `-z` guard keeps the old behaviour if the var is unset. ++ if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \ ++ && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then + echo " Skipping: $base (up to date)" + continue + fi +diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh +index 18d4357..8e38209 100755 +--- a/llamafile/cuda.sh ++++ b/llamafile/cuda.sh +@@ -293,10 +293,29 @@ collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_Q + echo " Sources: $NUM_SOURCES .cu files" + echo "" + ++# opencoti-hook: cuda-build-ccache (bug-858) — see docs/protocols/LLAMAFILE_UPGRADE.md. ++# The CUDA kernels live in .cuh HEADERS (fattn-vec.cuh, fattn-mma-f16.cuh, ...); the ++# template-instance .cu stubs that #include them essentially never change. The per-TU ++# incremental skip in build-functions.sh compared each object ONLY to its .cu file, so editing a ++# .cuh recompiled NOTHING → a silently STALE DSO (bug-858; same class as the host bug-167 -MMD ++# fix, which was never applied to this hand-rolled CUDA loop). Fix: (1) route every TU compile ++# through ccache, whose include-manifest invalidates the cached object when the .cu OR any ++# included header changes; (2) export the newest header mtime in the ggml tree so the no-ccache ++# fallback skip is header-aware. Linking (link_shared_library) stays on raw $NVCC — ccache caches ++# compiles only. ++CUDA_CC="$NVCC" ++if command -v ccache >/dev/null 2>&1; then ++ CUDA_CC="ccache $NVCC" ++ echo " ccache: ON for CUDA compilation (header-aware via include manifest)" ++else ++ echo " ccache: NOT FOUND — install ccache for fast + correct incremental CUDA rebuilds" ++fi ++export OPENCOTI_CUDA_DEPS_NEWEST=$(find "$GGML_CUDA_DIR/../.." \( -name '*.cuh' -o -name '*.h' -o -name '*.hpp' \) -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) ++ + START_TIME=$(date +%s) + + # Compile GPU sources +-compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" ++compile_gpu_sources_parallel "$CUDA_CC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" + + COMPILE_TIME=$(date +%s) + echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds" diff --git a/patches/0093-mtp-dualctx-fused-nextn.patch b/patches/0093-mtp-dualctx-fused-nextn.patch new file mode 100644 index 0000000000000000000000000000000000000000..d0f524d480448d9260063af5180999a9aad74ce0 --- /dev/null +++ b/patches/0093-mtp-dualctx-fused-nextn.patch @@ -0,0 +1,2873 @@ +From: opencoti +Subject: [PATCH 0093] bug-858 MTP parity: dual-ctx assistant port + fused NextN + verify-path fixes + +Wholesale alignment of speculative MTP to upstream b9859 (#590/#607/#599/#609): +- gemma4-assistant dual-context execution (ctx_dft sharing the target KV via + cparams.ctx_other / mem_other + share selector; OPENCOTI_MTP_DUAL_CTX gate), + draft_mtp impl with pre-norm embedding tap/harvest, defer-gather boot fix. +- fused NextN greedy N-step draft (build_one_step + decode_mtp_fused_nextn on a + dedicated sched_nextn; can_reuse base-drift + ensure_cleared aliased-KV fixes). +- bug-2103: mmvf ne11 cap raised to 8 under TinyBLAS (n3 verify GEMV cliff). +- bug-2094 diag + rolling-KV visibility hunks ride along in llama-kv-cache.cpp + (former 0091-rolling-kv-diag-visibility, folded here during the bug-2108 + chain repair). + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +index 89bd9dd..3fb1518 100644 +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1455,7 +1455,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + // opencoti F5 M7 Rolling KV — Rung 0 auto residency — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--vram-target"}, "MiB", +- string_format("rolling-kv: VRAM budget cap in MiB for auto KV residency (used with --headinfer-gpu-heads-frac auto). 0 = use all free VRAM minus a compute reserve (maximize). (default: %d)", params.vram_target_mib), ++ string_format("rolling-kv: TOTAL GPU VRAM cap in MiB (weights + KV + compute), NOT a KV-only budget. Auto KV residency keeps as much KV on-device as fits under this cap after weights are loaded; the overflow streams from host. Set it to your card's size to simulate/bound that card. 0 = use all free VRAM minus a compute reserve (maximize). (default: %d)", params.vram_target_mib), + [](common_params & params, int value) { + params.vram_target_mib = value; + } +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index b688492..3bb6aa2 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -303,9 +303,12 @@ struct common_params_speculative_draft { + int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding + int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding + +- // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): draft block size B produces +- // B-1 draft tokens per round. Default 3 (= 2 chained MTP draft steps). +- int32_t draft_block_size = 3; ++ // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): OPTIONAL explicit draft-depth ceiling. ++ // 0 (default) = draft depth follows --spec-draft-n-max (n_max); the MTP head re-runs ++ // autoregressively per step (decode_mtp_sync/fused), so depth is NOT limited to a fixed ++ // block size. bug-858: the old default 3 hard-capped ours at 2 drafts/round and blocked ++ // n-max scaling vs upstream b9859. When set >1, caps draft depth at draft_block_size-1. ++ int32_t draft_block_size = 0; + + float p_split = 0.1f; // speculative decoding split probability + float p_min = 0.0f; // minimum speculative decoding probability (greedy) +diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp +index 7faeac7..ee7962f 100644 +--- a/llama.cpp/common/speculative.cpp ++++ b/llama.cpp/common/speculative.cpp +@@ -481,6 +481,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + + int32_t n_embd = 0; + ++ // opencoti bug-858 dual-context MTP: true for the gemma4 assistant (ctx_dft shares the target's ++ // KV via ctx_other). Derived in the ctor. When set, process() skips the ctx_dft catch-up decode ++ // (the shared cache is already populated by the target's own decode) and draft() pins all draft ++ // tokens to the same position dp.n_past (KV-less: no per-step cell growth). Mirrors upstream ++ // b9859 common_speculative_impl_draft_mtp::is_mem_shared. false = Qwen NextN (byte-identical). ++ bool is_mem_shared = false; ++ + // Per-sequence cross-batch carryover: pair (h_p, x_{p+1}) at MTP pos p+1. + // The last h-row of one process() call needs the first token of the NEXT + // call to pair with, so it's stashed here until that next call fires. +@@ -507,7 +514,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + auto * ctx_dft = this->params.ctx_dft; + GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set"); + +- n_embd = llama_model_n_embd(llama_get_model(ctx_dft)); ++ // opencoti bug-858 dual-context MTP: the gemma4 assistant runs as ctx_dft with ++ // ctx_other=ctx_tgt (shared KV). Detect it here; every branch below keys off this flag. ++ is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; ++ ++ // n_embd = width of an h row (target backbone hidden). For the assistant the ctx_dft model ++ // is the assistant, whose n_embd_out() == the target n_embd (validated at load); for Qwen ++ // NextN the ctx_dft model IS the target, so keep the original accessor (byte-identical). ++ n_embd = is_mem_shared ++ ? llama_model_n_embd_out(llama_get_model(ctx_dft)) ++ : llama_model_n_embd(llama_get_model(ctx_dft)); + + LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); +@@ -550,7 +566,19 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + } + } + +- llama_set_embeddings_pre_norm(ctx_tgt, true, /*masked*/ false); ++ // opencoti bug-858 losslessness (2026-07-05): the target tap must not change the target's graph. ++ // The unmasked pre-norm tap engages gemma4.cpp's mtp_defer_out_ids (late row-strip: the last ++ // layer + output_norm run on ALL verify rows instead of the output rows) — structurally equal to ++ // upstream b9859's defer, but ours' GEMM kernels are batch-shape-sensitive (#495), so the deferred ++ // graph flips near-tie argmax vs plain decode (gate4/5: fib DIFF@2, forces DIFF@126; the facade, ++ // which never defers, is clean). Fix: for the shared-KV assistant set the target tap MASKED — ++ // masked keeps the normal early strip (target graph BIT-IDENTICAL to plain decode; t_h_pre_norm ++ // aliases the always-captured t_embd) and captures n_outputs rows with the output_ids-translated ++ // _ith accessor. Every verify token has logits=true, and prefill only ever consumes its last row ++ // (also an output row), so all consumed values equal upstream's all-rows defer capture — see the ++ // logits-rows harvest in process(). Qwen NextN (is_mem_shared==false) keeps the dense unmasked ++ // tap (its catch-up decode reads the whole shifted prefill buffer) — byte-identical. ++ llama_set_embeddings_pre_norm(ctx_tgt, true, /*masked*/ is_mem_shared); + llama_set_embeddings_pre_norm(ctx_dft, true, /*masked*/ true); + + pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); +@@ -591,7 +619,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + } + auto * ctx_dft = this->params.ctx_dft; + const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); +- if (pos_max < N - 1) { ++ // opencoti bug-858 dual-context MTP: for the shared-KV assistant, ctx_dft's positions are ++ // managed by the TARGET (no catch-up decode), so this warning is spurious — suppress it. ++ if (pos_max < N - 1 && !is_mem_shared) { + LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " + "process() hook may not have run on every prefill ubatch " + "(need_embd / logits=1 on every prompt position?). " +@@ -634,6 +664,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + ++ // opencoti bug-858 dual-context MTP: with a shared KV (gemma4 assistant) the ctx_dft cache is ++ // already populated by the TARGET's own decode (A2/A3 cell sharing), so skip the catch-up ++ // decode entirely and go straight to harvesting the target's verify hidden rows below. ++ // Mirrors upstream b9859 process() (`if (!is_mem_shared) { ...catch-up... }`). Qwen NextN ++ // (is_mem_shared==false) runs the catch-up exactly as before — byte-identical. ++ if (!is_mem_shared) { + common_batch_clear(batch); + + for (int k = 0; k < n_tokens; ++k) { +@@ -677,19 +713,44 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); + return false; + } ++ } // end if (!is_mem_shared): shared-KV assistant skips the ctx_dft catch-up decode + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (i_batch_end[seq_id] < 0) { + continue; + } + +- const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; ++ // opencoti bug-858 losslessness: with the MASKED target tap (is_mem_shared) only rows with ++ // logits set exist in the capture (verify batches: every row; prefill ubatches: the last row). ++ // Harvest exactly those; accept()'s row indexing is unchanged because every verify token has ++ // logits=true, and prefill only ever consumes pending_h (the last row). Qwen NextN (dense ++ // unmasked tap) takes the skip_row==false path on every row — byte-identical to before. ++ auto skip_row = [&](int32_t k) { ++ return is_mem_shared && batch_in.logits && !batch_in.logits[k]; ++ }; ++ ++ int32_t n_rows = 0; ++ for (int32_t k = i_batch_beg[seq_id]; k <= i_batch_end[seq_id]; ++k) { ++ if (skip_row(k)) { ++ continue; ++ } ++ n_rows++; ++ } ++ if (n_rows <= 0) { ++ continue; ++ } + verify_h_rows[seq_id] = n_rows; + verify_h[seq_id].resize((size_t) n_rows * n_embd); + +- for (int32_t i = 0; i < n_rows; ++i) { +- const float * h = llama_get_embeddings_pre_norm_ith(ctx_tgt, i_batch_beg[seq_id] + i); +- std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes); ++ int32_t r = 0; ++ for (int32_t k = i_batch_beg[seq_id]; k <= i_batch_end[seq_id]; ++k) { ++ if (skip_row(k)) { ++ continue; ++ } ++ // harvest via the pre-norm tap for BOTH paths (== upstream llama_get_embeddings_nextn_ith) ++ const float * h = llama_get_embeddings_pre_norm_ith(ctx_tgt, k); ++ std::memcpy(verify_h[seq_id].data() + (size_t) r * n_embd, h, row_bytes); ++ r++; + } + + std::memcpy(pending_h[seq_id].data(), +@@ -711,6 +772,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + const float * h_row = nullptr; + const size_t row_bytes = (size_t) n_embd * sizeof(float); + ++ // opencoti #590/bug-858 MTP profiling — gated, byte-identical when unset. + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + auto & dp = dparams[seq_id]; + +@@ -734,6 +796,55 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + return; + } + ++ // opencoti fused-NextN (#590 / bug-858): the seed decode above put id_last into ctx_dft's ++ // cache at n_past. When OPENCOTI_MTP_FUSED_NEXTN is set, emit all N drafts from ONE fused ++ // graph (single host sync) instead of the per-step AR loop below — the throughput fix. ++ // GREEDY-ONLY: no p_min early-stop (n_steps = n_max). Default OFF until the S5 gate proves ++ // acceptance holds vs the AR path. See docs/features/fused_nextn_mtp.md. ++ static const bool fused_nextn = [] { ++ const char * e = getenv("OPENCOTI_MTP_FUSED_NEXTN"); ++ return e && e[0] && e[0] != '0'; ++ }(); ++ // opencoti bug-858 dual-context MTP: the fused-NextN kernel is Qwen-NextN-specific ++ // (llama_decode_mtp_fused_nextn); never take it for the shared-KV assistant. ++ if (fused_nextn && params.n_max >= 2 && !is_mem_shared) { ++ for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { ++ if (!drafting[seq_id]) { ++ continue; ++ } ++ auto & dp = dparams[seq_id]; ++ llama_memory_t mem = llama_get_memory(ctx_dft); ++ llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; ++ if (attn_pos < 0) { ++ attn_pos = 0; ++ } ++ const int32_t n_steps = params.n_max; ++ std::vector out((size_t) n_steps, 0); ++ const int32_t rc = llama_decode_mtp_fused_nextn( ++ ctx_dft, seq_id, attn_pos, dp.id_last, pending_h[seq_id].data(), ++ n_steps, out.data(), /*out_h_prev_last=*/ nullptr); ++ if (rc != 0) { ++ LOG_WRN("%s: fused NextN draft failed rc=%d seq_id=%d\n", __func__, (int) rc, (int) seq_id); ++ continue; ++ } ++ auto & result = *dp.result; ++ for (int32_t k = 0; k < n_steps; ++k) { ++ result.push_back(out[(size_t) k]); ++ } ++ } ++ for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { ++ auto & dp = dparams[seq_id]; ++ if (!dp.drafting) { ++ continue; ++ } ++ if (dp.result->size() < (size_t) params.n_min) { ++ dp.result->clear(); ++ } ++ last_n_drafted[seq_id] = (uint16_t) dp.result->size(); ++ } ++ return; ++ } ++ + int i = 0; + + while (n_drafting > 0) { +@@ -748,23 +859,39 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + + auto * smpl = smpls[seq_id].get(); + +- common_sampler_sample(smpl, ctx_dft, i_batch, true); ++ // opencoti #590/bug-858: on-device greedy draft-token fast path. When p_min<=0 (the ++ // default — no confidence early-stop is configured), the full-vocab common_sampler_sample ++ // (logit D2H + CPU softmax/sort over ~152k vocab) is MEASURED as 0.98 ms/step = 100% of ++ // the draft-loop host tax (vs 0.14 ms decode-launch, 0.003 ms hidden). The draft sampler ++ // is deterministic-argmax and the graph already published that argmax on-device ++ // (qwen35 t_argmax), so read it back (4 B) and skip the host sampler entirely. p_min>0 ++ // falls through to the exact host path (which computes the top-token probability). ++ llama_token id = -1; ++ float id_p = 1.0f; // on-device greedy is unconditionally accepted when p_min<=0 ++ // opencoti bug-858 dual-context MTP: the shared-KV assistant graph publishes no ++ // on-device argmax (t_logits + t_h_pre_norm only), so host-sample it (matches ++ // upstream b9859, which always common_sampler_sample's the draft). Byte-identical ++ // for Qwen NextN (is_mem_shared==false). ++ if (params.p_min <= 0.0f && !is_mem_shared) { ++ id = llama_get_draft_greedy_ith(ctx_dft, i_batch); // includes synchronize() = GPU-wait ++ } ++ if (id < 0) { ++ // exact host sampler: p_min>0, or the graph published no on-device argmax ++ common_sampler_sample(smpl, ctx_dft, i_batch, true); ++ const auto * cur_p = common_sampler_get_candidates(smpl, true); ++ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { ++ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", ++ seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, ++ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); ++ } ++ id = cur_p->data[0].id; ++ id_p = cur_p->data[0].p; ++ } + h_row = llama_get_embeddings_pre_norm_ith(ctx_dft, i_batch); + ++i_batch; + +- const auto * cur_p = common_sampler_get_candidates(smpl, true); +- +- for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { +- LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", +- seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, +- common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); +- } +- +- // add drafted token for each sequence +- const llama_token id = cur_p->data[0].id; +- + // only collect very high-confidence draft tokens +- if (cur_p->data[0].p < params.p_min) { ++ if (id_p < params.p_min) { + drafting[seq_id] = false; + n_drafting--; + +@@ -784,7 +911,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + continue; + } + +- common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); ++ // opencoti bug-858 dual-context MTP: the shared-KV assistant is KV-less and pins ++ // EVERY draft token to dp.n_past (no per-step cell growth; all drafts attend the ++ // frozen target prefix) — this mirrors upstream b9859 draft()'s is_mem_shared branch ++ // and the HF gemma4_assistant reference. Qwen NextN advances the position per step ++ // (byte-identical to before). ++ const llama_pos draft_pos = is_mem_shared ? dp.n_past : dp.n_past + i + 1; ++ common_batch_add(batch, id, draft_pos, { seq_id }, true); + std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); + } + +@@ -814,6 +947,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + + last_n_drafted[seq_id] = (uint16_t) dp.result->size(); + } ++ + } + + void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { +@@ -832,10 +966,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { + } + + bool need_embd() const override { ++ // opencoti bug-858 losslessness (2026-07-04): NEVER put the target in plain embeddings mode. ++ // cparams.embeddings=true forces output_all=true (llama-context.cpp:2077), which changes the ++ // target's own decode graph and marks t_embd (lm_head input) as an output → perturbs the target's ++ // verify logits vs plain decode → near-tie argmax flips → spec != plain. Both A4B assistant and ++ // Qwen NextN harvest via the pre-norm tap now (matches upstream b9859, cparams.embeddings=false). + return false; + } + + bool need_embd_pre_norm() const override { ++ // opencoti bug-858: both paths harvest the pre-norm hidden from ctx_tgt (== upstream embeddings_nextn). + return true; + } + }; +@@ -1409,6 +1549,8 @@ struct common_speculative_impl_draft_assistant : public common_speculative_impl + // can select the last-accepted token's hidden state. The spec framework calls process() after the + // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which + // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. ++ // NOTE (bug-858): PRE-norm harvest was TESTED and REFUTED — it collapses accept to ~0.05 at ALL ++ // positions (incl short ctx). This drafter expects the post-norm hidden; do NOT swap to pre_norm. + bool process(const llama_batch & batch_in) override { + if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { + return true; +@@ -1463,8 +1605,12 @@ struct common_speculative_impl_draft_assistant : public common_speculative_impl + return; + } + +- const int32_t block = params.draft_block_size; +- const int32_t n_steps_raw = block > 1 ? block - 1 : 0; ++ // bug-858: draft depth follows --spec-draft-n-max (n_max) and the room left in context ++ // (dp.n_max = n_draft_max), NOT draft_block_size. The MTP head re-runs autoregressively per ++ // step (decode_mtp_sync/fused build a fresh single-step graph each step: argmax -> next ++ // last_token, h_post -> next h_prev), so depth is bounded by n_max, not a fixed block. The ++ // legacy draft_block_size-1 ceiling (default 3 -> 2) hard-capped ours at 2 drafts/round and ++ // blocked n-max scaling vs upstream b9859 (which drafts to n_max autoregressively) -> REMOVED. + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + auto & dp = dparams[seq_id]; +@@ -1488,11 +1634,10 @@ struct common_speculative_impl_draft_assistant : public common_speculative_impl + continue; // empty result -> server falls back to single-token verify + } + +- int32_t n_steps = n_steps_raw; ++ int32_t n_steps = params.n_max > 0 ? params.n_max : 1; + if (dp.n_max > 0) { + n_steps = std::min(n_steps, dp.n_max); + } +- n_steps = std::min(n_steps, params.n_max); + if (n_steps <= 0) { + prev_n_acc_at_draft[seq_id] = n_acc_drafts; + continue; +@@ -1789,7 +1934,15 @@ common_speculative * common_speculative_init(common_params_speculative & params, + break; + } + case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter +- impls.push_back(std::make_unique(config.params, n_seq)); ++ // opencoti bug-858 dual-context MTP: when the server created a real ctx_dft FROM the ++ // assistant model (ctx_other=ctx_tgt; --spec-type draft-assistant + OPENCOTI_MTP_DUAL_CTX), ++ // run the shared-KV draft_mtp driver (upstream b9859 is_mem_shared). Otherwise fall ++ // back to the proven single-context in-target assistant (ctx_dft == nullptr). ++ if (config.params.draft.ctx_dft != nullptr) { ++ impls.push_back(std::make_unique(config.params, n_seq)); ++ } else { ++ impls.push_back(std::make_unique(config.params, n_seq)); ++ } + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { +diff --git a/llama.cpp/ggml/src/ggml-backend.cpp b/llama.cpp/ggml/src/ggml-backend.cpp +index c2601b1..752e7ca 100644 +--- a/llama.cpp/ggml/src/ggml-backend.cpp ++++ b/llama.cpp/ggml/src/ggml-backend.cpp +@@ -276,6 +276,14 @@ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_ten + GGML_ASSERT(backend); + GGML_ASSERT(tensor); + GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); ++ if (offset + size > ggml_nbytes(tensor)) { // opencoti bug-858 diag (REMOVE after root-cause) ++ fprintf(stderr, "TENSOR_GET_ASYNC_OOB: name='%s' type=%d ne=[%lld,%lld,%lld,%lld] nbytes=%zu offset=%zu size=%zu\n", ++ tensor->name, (int) tensor->type, ++ (long long) tensor->ne[0], (long long) tensor->ne[1], ++ (long long) tensor->ne[2], (long long) tensor->ne[3], ++ ggml_nbytes(tensor), offset, size); ++ fflush(stderr); ++ } + GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); + + if (backend->iface.get_tensor_async == NULL) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index e53ac9c..e63be74 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -42,13 +42,6 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const ggml_tensor * Q = dst->src[0]; + +- if constexpr (ncols2 <= 8) { +- if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) { +- ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); +- return; +- } +- } +- + if constexpr (ncols2 <= 16) { + if (Q->ne[1] <= 16/ncols2) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); +@@ -597,6 +590,13 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { + if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { ++ // opencoti bug-858 (2026-07-05): REVERTED the earlier <=2 experiment back to upstream's ++ // Q->ne[1]==1. Routing the n_q=2 verify to VEC (cols_per_block=2) did NOT make it bit-equal ++ // to the n_q=1 decode (different column tiling → different accumulation order) and measurably ++ // WORSENED dual-vs-plain losslessness at n_max=1 (gate4b: 0/4 prompts byte-equal vs 1/4 on ++ // upstream ==1 semantics; the earlier accept win attributed to this change was actually the ++ // output_all/masked-tap defect, fixed in common/speculative.cpp). Keep upstream semantics: ++ // VEC for decode only; verify (n_q>=2) takes MMA exactly like b9859. + if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) { + return BEST_FATTN_KERNEL_VEC; + } +@@ -1328,8 +1328,27 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + // all_f16 fast path (the memcpy lift) but cheap to compute. + const size_t ts_k = ggml_type_size(K->type); + const size_t ts_v = ggml_type_size(V->type); +- ggml_cuda_pool_alloc slotK(pool, (size_t)n_slots * k_slot); +- ggml_cuda_pool_alloc slotV(pool, (size_t)n_slots * v_slot); ++ // opencoti-hook: f5-rolling-kv #586 — PERSISTENT staging ring (was ++ // ggml_cuda_pool_alloc). The pool re-handed the SAME slot addresses every ++ // op/layer, which is the sole reason the coarse per-op cs_sync barrier ++ // existed (#587/bug-255): copy_stream had to wait for ALL of the previous ++ // layer's compute before ANY lift, serialising the tail DMA across every ++ // streaming layer — the S0 overflow-decode cliff (docs/features/rolling_kv.md). ++ // A persistent ring gives STABLE slot addresses across ops, so the existing ++ // per-slot compute_done events gate cross-op WAR at slot granularity and the ++ // barrier is dropped below. SAME n_slots × slot_size footprint → no extra ++ // VRAM (the serving target is VRAM-bound); realloc-on-grow as tile_kv_full / ++ // heads grow during decode. Single-device static (matches the existing ++ // single-static copy_done/compute_done event pool; the overlap path is ++ // single-device in practice). ++ static char * g_slotK = nullptr; static size_t g_capK = 0; ++ static char * g_slotV = nullptr; static size_t g_capV = 0; ++ const size_t need_k = (size_t)n_slots * k_slot; ++ const size_t need_v = (size_t)n_slots * v_slot; ++ if (g_capK < need_k) { if (g_slotK) CUDA_CHECK(cudaFree(g_slotK)); CUDA_CHECK(cudaMalloc((void**)&g_slotK, need_k)); g_capK = need_k; } ++ if (g_capV < need_v) { if (g_slotV) CUDA_CHECK(cudaFree(g_slotV)); CUDA_CHECK(cudaMalloc((void**)&g_slotV, need_v)); g_capV = need_v; } ++ char * const slotK_base = g_slotK; ++ char * const slotV_base = g_slotV; + + // R2-b S3b (#312): with >=2 slots the op runs a double-buffer ping-pong — the + // per-tile lift is issued on a dedicated copy_stream while FA/lse for the +@@ -1348,7 +1367,20 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + if (use_overlap) { + const int cs_idx = GGML_CUDA_MAX_STREAMS - 1; // fixed, collision-free + GGML_ASSERT(cs_idx >= 1); +- cudaStream_t cs = ctx.stream(ctx.device, cs_idx); ++ // opencoti-hook: f5-rolling-kv — bug-1841 (#588/#587). At DECODE (n_q==1) collapse ++ // the copy/compute double-buffer onto ctx.stream(): the cross-stream ++ // copy_done/compute_done waits below CANNOT be captured into the decode CUDA graph ++ // ("dependency created on uncaptured work in another stream") — which forced graphs ++ // OFF and made the streaming op launch/sync-bound (per-tile × 48-layer event ++ // serialization, PCIe-independent ~1 tps). On a single stream the tile lift → FA → ++ // lse are stream-ordered (every real dependency implicit; the slot ping-pong's WAR ++ // is satisfied because compute(t) is enqueued before lift(t+2) reuses its slot), so ++ // the whole op is graph-capturable and the 48-layer decode is amortized by graph ++ // replay. n_q>1 (prefill) keeps the true copy/compute overlap (compute-heavy, never ++ // graph-captured). Trades the decode copy/compute overlap — cheap on the fast ++ // serving-target PCIe — for graph amortization, the actual decode-tps lever. ++ const bool single_stream = (n_q == 1); ++ cudaStream_t cs = single_stream ? stream : ctx.stream(ctx.device, cs_idx); + + // Pre-created event pool (>= S3a slot cap 4). cudaEventDisableTiming; + // created ONCE during the eager warmup (op dispatch is single-thread, +@@ -1364,20 +1396,19 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + ev_init = true; + } + +- // bug-255: the slot pool buffers are reclaimed/re-handed across op +- // invocations (every layer reuses the same pool addresses). In S3a +- // copy+FA shared one stream so layer L+1's copy was implicitly ordered +- // after layer L's FA. With the copy moved to copy_stream, layer L+1's +- // lift could overwrite a slot while layer L's FA (on the compute +- // stream) is still reading it → cross-op WAR. Order copy_stream after +- // all prior compute-stream work so a reused slot is never clobbered +- // mid-read. (One event per op entry; intra-op order is the ping-pong.) +- static cudaEvent_t cs_sync = nullptr; +- if (cs_sync == nullptr) { +- CUDA_CHECK(cudaEventCreateWithFlags(&cs_sync, cudaEventDisableTiming)); +- } +- CUDA_CHECK(cudaEventRecord(cs_sync, stream)); +- CUDA_CHECK(cudaStreamWaitEvent(cs, cs_sync, 0)); ++ // opencoti-hook: f5-rolling-kv #587 — cross-layer run-ahead. The coarse ++ // per-op cs_sync barrier (record on `stream`, wait on `cs`) used to force ++ // copy_stream to wait for ALL of the previous layer's compute before any ++ // lift — because the pool re-handed the same slot addresses each op ++ // (bug-255). With the PERSISTENT ring (#586, above) slot addresses are ++ // stable, so per-slot compute_done events gate cross-op WAR at slot ++ // granularity: slot_live[s] marks a slot holding an un-retired writer from ++ // a prior op/layer, and each lift into slot s waits compute_done[s] iff ++ // slot_live[s]. Dropping the whole-stream barrier lets layer L+1's tail ++ // DMA overlap layer L's compute — inside the replayed cuda-graph, exactly ++ // where S0 located the cliff cost. Byte-identical (only a conservative ++ // barrier is removed; the per-slot edges preserve every real dependency). ++ static bool slot_live[4] = {}; + + // Lift tile t's K/V into its slot on the copy_stream (cudaMemcpyDefault + // → UVA H2D/D2D). Same packed 1-D-per-(b,h) layout as the S3a path. +@@ -1402,8 +1433,8 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + const int slot = t % n_slots; + const size_t k_nb2 = (size_t)this_kv * k_row; + const size_t v_nb2 = (size_t)this_kv * v_row; +- char * sK = slotK.ptr + (size_t)slot * k_slot; +- char * sV = slotV.ptr + (size_t)slot * v_slot; ++ char * sK = slotK_base + (size_t)slot * k_slot; ++ char * sV = slotV_base + (size_t)slot * v_slot; + if (!all_f16) { + // S3d dequant-on-lift: ONE nc-converter launch per tile reads the + // (head_dim × this_kv × n_head_kv × n_b) source sub-block at its +@@ -1422,17 +1453,45 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + (int64_t)(tV->nb[1]/ts_v), (int64_t)(tV->nb[2]/ts_v), (int64_t)(tV->nb[3]/ts_v), cs); + } else + for (int bb = 0; bb < n_b; ++bb) { ++ // opencoti-hook: f5-rolling-kv #586 — bulk H2D + on-device repack. ++ // The per-head strided 2D copy issues n_head_kv rows of only k_row ++ // (=head_dim·2B = 256B) bytes each; over PCIe those tiny strided ++ // transfers waste bandwidth (measured ~3.1 GB/s vs the 6.5 GB/s ++ // solidPC ceiling — ~2× loss; the loss is worse at higher BW where ++ // small rows can't saturate the link). The host tail is CONTIGUOUS ++ // in cell-major [n_embd_gqa × this_kv] (nb[1]=cell stride), so for a ++ // HOST source bulk-copy the whole block H2D once (full bandwidth) ++ // into a device scratch, then do the cell→head repack ON-DEVICE ++ // (D2D strided ≈ GPU bandwidth, ~free). Byte-identical: the slot ends ++ // up with the exact same packed bytes. A device window source is ++ // already D2D, so keep its in-place strided path (no scratch). ++ const bool bulk = use_2d && tr.host; ++ const size_t kblk = (size_t)this_kv * tK->nb[1]; // contiguous cell-major span ++ const size_t vblk = (size_t)this_kv * tV->nb[1]; ++ ggml_cuda_pool_alloc scrK(pool, bulk ? kblk : 1); ++ ggml_cuda_pool_alloc scrV(pool, bulk ? vblk : 1); ++ const char * baseK; ++ const char * baseV; ++ if (bulk) { ++ const char * hK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1]; ++ const char * hV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1]; ++ CUDA_CHECK(cudaMemcpyAsync(scrK.ptr, hK, kblk, cudaMemcpyDefault, cs)); // one full-BW H2D ++ CUDA_CHECK(cudaMemcpyAsync(scrV.ptr, hV, vblk, cudaMemcpyDefault, cs)); ++ baseK = scrK.ptr; // scratch is cell-major, cell `loc` at offset 0 ++ baseV = scrV.ptr; ++ } else { ++ baseK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1]; ++ baseV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1]; ++ } + for (int hh = 0; hh < n_head_kv; ++hh) { + const size_t dstoff = (size_t)bb*n_head_kv + hh; +- const char * srcK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)hh*tK->nb[2] + (size_t)loc*tK->nb[1]; +- const char * srcV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)hh*tV->nb[2] + (size_t)loc*tV->nb[1]; ++ const char * srcK = baseK + (size_t)hh*tK->nb[2]; // loc already folded into base ++ const char * srcV = baseV + (size_t)hh*tV->nb[2]; + if (use_2d) { +- // S3c: strided source (pinned-host CPU-half, OR the 3c-5 +- // permuted device window — all heads per cell). Key rows +- // strided by nb[1]; the 2D copy packs them into the +- // contiguous slot (dst pitch == width == row), byte- +- // equivalent to the device 1-D path. cudaMemcpyDefault +- // (UVA) reads host or device transparently. ++ // repack cell→head-major: rows strided by nb[1], packed into ++ // the contiguous slot (dst pitch == width == row). For a bulk ++ // host tail this is now D2D (scratch→slot, fast); for a device ++ // window it is the original in-place strided D2D. + CUDA_CHECK(cudaMemcpy2DAsync( + sK + dstoff*k_nb2, k_row, srcK, tK->nb[1], k_row, this_kv, cudaMemcpyDefault, cs)); + CUDA_CHECK(cudaMemcpy2DAsync( +@@ -1446,8 +1505,10 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + }; + + // Prologue: stage tile 0 so its compute can start as soon as it lands. ++ // #587: if slot 0 still holds a prior op's writer, wait its compute first. ++ if (!single_stream && slot_live[0]) CUDA_CHECK(cudaStreamWaitEvent(cs, compute_done[0], 0)); + lift(0); +- CUDA_CHECK(cudaEventRecord(copy_done[0], cs)); ++ if (!single_stream) CUDA_CHECK(cudaEventRecord(copy_done[0], cs)); + + for (int t = 0; t < n_tiles; ++t) { + const int slot = t % n_slots; +@@ -1457,11 +1518,13 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + // compute to release it (compute_done was recorded n_slots tiles ago). + if (t + 1 < n_tiles) { + const int nslot = (t + 1) % n_slots; +- if (t + 1 >= n_slots) { ++ // #587: wait if the slot is reused within this op (t+1>=n_slots) ++ // OR still holds a prior op/layer's un-retired writer (slot_live). ++ if (!single_stream && (t + 1 >= n_slots || slot_live[nslot])) { + CUDA_CHECK(cudaStreamWaitEvent(cs, compute_done[nslot], 0)); + } + lift(t + 1); +- CUDA_CHECK(cudaEventRecord(copy_done[nslot], cs)); ++ if (!single_stream) CUDA_CHECK(cudaEventRecord(copy_done[nslot], cs)); + } + + // Compute tile t on ctx.stream(): wait for its copy, then FA + lse. +@@ -1473,9 +1536,9 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + const int kv0 = trc.global_kv0; + const int this_kv = trc.this_kv; + float * lse_t = lse_tiles.ptr + (size_t)t * n_rows; +- char * sK = slotK.ptr + (size_t)slot * k_slot; +- char * sV = slotV.ptr + (size_t)slot * v_slot; +- CUDA_CHECK(cudaStreamWaitEvent(stream, copy_done[slot], 0)); ++ char * sK = slotK_base + (size_t)slot * k_slot; ++ char * sV = slotV_base + (size_t)slot * v_slot; ++ if (!single_stream) CUDA_CHECK(cudaStreamWaitEvent(stream, copy_done[slot], 0)); + + if (this_kv > 0) { + const size_t k_nb2 = (size_t)this_kv * k_row; +@@ -1538,13 +1601,28 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + ggml_cuda_flash_attn_ext(ctx, &dst_t); + + if (!decode_lse) { +- streaming_lse_kernel<<>>( +- (const char *)Q->data, sK, +- mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, +- lse_t, scale, logit_softcap, this_kv, gqa, head_dim, n_head, n_q, +- Q->nb[1], Q->nb[2], Q->nb[3], k_row, k_nb2, k_nb2 * n_head_kv, +- mask ? mask->nb[0] : 0, mask ? mask->nb[1] : 0); +- CUDA_CHECK(cudaGetLastError()); ++ // opencoti F5 bug-1840 fix (b) DIAGNOSTIC — OPENCOTI_LSE_NOOP skips ++ // the per-tile streaming_lse recompute (a low-parallelism ++ // <<>> rescan run once per tile because Qwen/head_dim≤256 ++ // decode picks the finalize-less VEC FA → no dst_lse). Skipping it ++ // CORRUPTS the online-softmax combine (needle lost) — it exists ONLY ++ // to measure, via real decode tps, how much of the streaming-spill ++ // cost is this recompute vs the per-tile FA itself. If the tps ceiling ++ // justifies it, the real fix is a key-tiled parallel lse kernel (or ++ // emitting lse from the VEC FA). Default-off ⇒ byte-identical. ++ static const bool lse_noop = [] { ++ const char * e = getenv("OPENCOTI_LSE_NOOP"); ++ return e != nullptr && atoi(e) == 1; ++ }(); ++ if (!lse_noop) { ++ streaming_lse_kernel<<>>( ++ (const char *)Q->data, sK, ++ mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, ++ lse_t, scale, logit_softcap, this_kv, gqa, head_dim, n_head, n_q, ++ Q->nb[1], Q->nb[2], Q->nb[3], k_row, k_nb2, k_nb2 * n_head_kv, ++ mask ? mask->nb[0] : 0, mask ? mask->nb[1] : 0); ++ CUDA_CHECK(cudaGetLastError()); ++ } + } + } else { + // Empty tile (this_kv==0): the kernel writes lse=-inf and never +@@ -1555,7 +1633,8 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + Q->nb[1], Q->nb[2], Q->nb[3], k_row, 0, 0, 0, 0); + CUDA_CHECK(cudaGetLastError()); + } +- CUDA_CHECK(cudaEventRecord(compute_done[slot], stream)); ++ if (!single_stream) CUDA_CHECK(cudaEventRecord(compute_done[slot], stream)); ++ slot_live[slot] = true; // #587: slot holds an un-retired writer for the next op/layer + } + } else + // S3a: single compute stream. The per-tile slot lift, the FA over the +@@ -1569,8 +1648,8 @@ void ggml_cuda_streaming_flash_attn(ggml_backend_cuda_context & ctx, ggml_tensor + const int kv0 = t * tile_kv_full; + const int this_kv = kv0 >= n_kv ? 0 : min(tile_kv_full, n_kv - kv0); + float * lse_t = lse_tiles.ptr + (size_t)t * n_rows; +- char * sK = slotK.ptr + (size_t)slot * k_slot; +- char * sV = slotV.ptr + (size_t)slot * v_slot; ++ char * sK = slotK_base + (size_t)slot * k_slot; ++ char * sV = slotV_base + (size_t)slot * v_slot; + + if (this_kv > 0) { + const size_t k_nb2 = (size_t)this_kv * k_row; +diff --git a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +index 4db22f3..bad56a5 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +@@ -3344,6 +3344,23 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { + continue; + } + ++ // opencoti-hook: f5-rolling-kv — bug-1841 (#588/#587). The POSITION_WINDOW streaming FA op ++ // (ggml_cuda_streaming_flash_attn, fattn.cu) uses a copy/compute double-buffer ONLY at ++ // prefill (n_q>1): a SEPARATE copy stream + cross-stream copy_done/compute_done event ++ // waits, which cannot be captured into a CUDA graph ("dependency created on uncaptured ++ // work in another stream", fattn.cu:1528). At DECODE (n_q==1, src[0]->ne[1]==1) the op ++ // now runs single-stream (all lift/FA/lse on ctx.stream(), no cross-stream events) → ++ // fully graph-capturable, which is the actual decode-tps lever (the streaming path was ++ // launch/sync-bound, PCIe-independent, with graphs off). So disable graphs ONLY for the ++ // prefill overlap path (n_q>1, which is never graph-captured anyway — belt-and-suspenders); ++ // decode keeps CUDA graphs. See buglog bug-1841, docs/features/rolling_kv.md. ++ if (node->op == GGML_OP_STREAMING_FLASH_ATTN && node->src[0] && node->src[0]->ne[1] > 1) { ++ use_cuda_graph = false; ++#ifndef NDEBUG ++ GGML_LOG_DEBUG("%s: disabling CUDA graphs due to rolling-KV streaming FA (prefill overlap)\n", __func__); ++#endif ++ } ++ + if (node->src[0] && node->src[0]->buffer && ggml_backend_buft_is_cuda_split(node->src[0]->buffer->buft)) { + use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture + #ifndef NDEBUG +@@ -4617,7 +4634,41 @@ static enum ggml_status GGML_CALL ggml_backend_cuda_graph_compute(ggml_backend_t + if (graph_compatible) { + const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph); + +- if (!graph->warmup_complete) { ++ // opencoti F5 bug-1840 fix (a) — graph-live decode. The stock 0.10.3 ++ // warmup machine only engages graphs after TWO consecutive calls with ++ // NO property change, and RESETS to eager on any subsequent change. ++ // During decode the KV cache ne[1] grows one cell/step, so ++ // ggml_cuda_graph_update_required reports "changed" every step → ++ // warmup never completes → graphs never engage (measured: ++ // cudaGraphLaunch=0 across every decode profile). Mainline llama.cpp ++ // instead keeps graphs live across a growing KV via cudaGraphExecUpdate ++ // (already implemented here as ggml_cuda_graph_update_executable, 3440: ++ // cudaGraphExecUpdate + topology-change re-instantiate fallback). Under ++ // OPENCOTI_GRAPH_LIVE, complete warmup after ONE stabilization call, then ++ // stay on graphs and drive recapture-record + exec-update on each property ++ // change instead of dropping to eager. graph_key (nodes[0]) is stable ++ // across steps when llama.cpp reuses the graph result (can_reuse), so the ++ // per-step cost is a CPU recapture-record + cheap exec-update, not a ++ // re-instantiate. Default-off ⇒ byte-identical to stock. ++ // opencoti #590/bug-858: default-ON was TESTED and REVERTED — the ++ // bs2 A/B (graph-live on vs off) showed NO MTP benefit (35B n2 ++ // 233≈234, n3 still collapsed 156≈158; spec cells inert) and a ++ // REGRESSION on base decode (35B 201 vs 212) + A4B n3 (189 vs 209). ++ // The n=3 spec collapse is NOT CUDA-graph thrash — it is per-step ++ // spec-loop overhead (host readback/sync in the un-fused draft loop); ++ // graph-live is the wrong lever. Keep the knob env-gated, default-off. ++ static const bool graph_live = [] { ++ const char * e = getenv("OPENCOTI_GRAPH_LIVE"); ++ return e != nullptr && atoi(e) == 1; ++ }(); ++ if (graph_live) { ++ if (!graph->warmup_complete) { ++ graph->warmup_complete = true; // one stabilization call, then engage ++ } else { ++ use_cuda_graph = true; ++ cuda_graph_update_required = properties_changed || graph->instance == nullptr; ++ } ++ } else if (!graph->warmup_complete) { + // Warmup: need at least 2 calls with no property change on the 2nd call + if (!properties_changed) { + graph->warmup_complete = true; +diff --git a/llama.cpp/ggml/src/ggml-cuda/mmvf.cu b/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +index 09d95f3..4051188 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/mmvf.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +@@ -800,6 +800,14 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 + switch (type) { + case GGML_TYPE_F32: + if (GGML_CUDA_CC_IS_NVIDIA(cc)) { ++#ifdef GGML_USE_TINYBLAS ++ // opencoti-hook: mtp-verify-mmvf (bug-858/#609) — upstream caps NVIDIA f32 at ++ // ne11<=3 because past that its cuBLAS fallback wins; TinyBLAS builds have no ++ // cuBLAS and the fallback is a single-block tinyblasGE (~174us vs ~1.2us for ++ // the fused-GDN dot GEMVs in a 4-token MTP verify batch => 35% of decode wall ++ // at spec depth 3). Use the kernel's real max (8, the non-NVIDIA default below). ++ return ne11 <= 8; ++#endif // GGML_USE_TINYBLAS + if (ampere_mma_available(cc)) { + return ne11 <= 3; + } +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index a0a435c..e6517be 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -424,6 +424,9 @@ extern "C" { + ggml_abort_callback abort_callback; + void * abort_callback_data; + ++ // opencoti bug-858: dual-context MTP — target context whose KV the draft context shares (nullptr if none) ++ struct llama_context * ctx_other; ++ + // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. + bool embeddings; // if true, extract embeddings (together with logits) + bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU +@@ -617,6 +620,8 @@ extern "C" { + + LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); + LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); ++ // opencoti bug-858: dual-context MTP — the target context this context shares KV with (nullptr if none) ++ LLAMA_API struct llama_context * llama_get_ctx_other(struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type + + LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); +@@ -1052,6 +1057,19 @@ extern "C" { + float * out_logits, + float * out_h_prev_last); + ++ // opencoti fused-NextN (#590): fuse N per-step NextN draft decodes into one graph on a ++ // DECODER_MTP (Qwen NextN) draft context. GREEDY-ONLY (no p_min early-stop). Returns 0 on ++ // success. last_token must already be resident in ctx's cache at attn_pos (seed decode first). ++ LLAMA_API int32_t llama_decode_mtp_fused_nextn( ++ struct llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 9dbc574..7d27212 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -155,6 +155,7 @@ llama_context::llama_context( + cparams.cb_eval_user_data = params.cb_eval_user_data; + + cparams.ctx_type = params.ctx_type; ++ cparams.ctx_other = params.ctx_other; // opencoti bug-858: dual-context MTP KV sharing + + // Initialize backend samplers here so they are part of the sampling graph + // before the reserve passes run later in this function. This avoids a later +@@ -398,6 +399,7 @@ llama_context::llama_context( + /*.type_v =*/ params.type_v, + /*.swa_full =*/ params.swa_full, + /*.ctx_type= */ cparams.ctx_type, ++ /*.mem_other=*/ cparams.ctx_other ? llama_get_memory(cparams.ctx_other) : nullptr, // opencoti bug-858 + }; + + // opencoti bug-1342: pass 1 forces a minimal resident window (measure mode). +@@ -498,6 +500,7 @@ llama_context::llama_context( + /*.type_v =*/ params.type_v, + /*.swa_full =*/ params.swa_full, + /*.ctx_type= */ cparams.ctx_type, ++ /*.mem_other=*/ cparams.ctx_other ? llama_get_memory(cparams.ctx_other) : nullptr, // opencoti bug-858 + }; + memory.reset(model.create_memory(params_mem2, cparams)); + +@@ -576,6 +579,8 @@ void llama_context::sched_reserve() { + + gf_res_prev.reset(new llm_graph_result(max_nodes)); + gf_res_reserve.reset(new llm_graph_result(max_nodes)); ++ // opencoti #590/bug-858: sched_nextn + gf_res_prev_nextn are created lazily in ++ // decode_mtp_fused_nextn (sized graph_max_nodes*n_steps) — see header note. + + sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload)); + +@@ -2012,7 +2017,17 @@ int llama_context::decode(const llama_batch & batch_inp) { + const auto & hparams = model.hparams; + + const int64_t n_vocab = vocab.n_tokens(); +- const int64_t n_embd = hparams.n_embd_inp(); ++ // opencoti bug-858 (dual-context MTP feed width): the gemma4-assistant drafter runs as ctx_dft ++ // (LLAMA_CONTEXT_TYPE_MTP) and consumes the TARGET's backbone hidden — n_embd_out wide (2816) — ++ // as its input embd, NOT its own n_embd (1024). n_embd_inp() returns 1024, so the batch→ubatch ++ // split (llama-batch.cpp: memcpy stride n_embd) would carry 1024-wide rows while ++ // llm_graph_input_embd::set_input writes n_embd_out(2816) → overread of uninitialised staging → ++ // partial NaN in inp_h → 0% accept. Size the batch by n_embd_out for such a context. Guarded to ++ // the genuine mismatch (n_embd_out_impl != n_embd), so normal models stay byte-identical. ++ const int64_t n_embd = (cparams.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_embd_out_impl > 0 && ++ hparams.n_embd_out_impl != hparams.n_embd) ++ ? (int64_t) hparams.n_embd_out() ++ : (int64_t) hparams.n_embd_inp(); + + // when computing embeddings, all tokens are output + const bool output_all = cparams.embeddings; +@@ -2836,7 +2851,13 @@ int32_t llama_context::decode_mtp_sync( + // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. + for (int32_t k = 0; k < n_steps; ++k) { + data->token[0] = last_token; +- data->pos[0] = attn_pos + 1 + (llama_pos) k; ++ // bug-858: the gemma4-assistant (is_mem_shared) drafts ALL steps from the SAME query ++ // position — the head is trained to predict multiple future tokens from one position, with ++ // the recurrence carried by the h_prev hidden chain (NOT the RoPE angle). Incrementing the ++ // position per step (attn_pos+1+k) hands the head a mistrained RoPE angle from step 1 on, ++ // degrading step-1+ drafts (aggregate accept ~0.69 vs upstream 0.79). Match b9859 draft_mtp ++ // is_mem_shared: common_batch_add(batch, id, dp.n_past, ...) — constant dp.n_past = attn_pos+1. ++ data->pos[0] = attn_pos + 1; + std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); + + llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +@@ -2978,8 +2999,12 @@ int32_t llama_context::decode_mtp_fused( + + data->token[0] = last_token; + std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); ++ // bug-858: constant query position for ALL draft steps (see decode_mtp_sync note). The ++ // gemma4-assistant (is_mem_shared) is trained to predict every future draft token from the ++ // SAME position (dp.n_past = attn_pos+1); recurrence lives in the h_prev chain, not the RoPE ++ // angle. The old attn_pos+1+k mistrained step-1+ RoPE -> accept 0.69 vs upstream 0.79. + for (int32_t k = 0; k < n_steps; ++k) { +- data->pos[k] = attn_pos + 1 + (llama_pos) k; ++ data->pos[k] = attn_pos + 1; + } + + llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +@@ -3016,6 +3041,192 @@ int32_t llama_context::decode_mtp_fused( + return 0; + } + ++// opencoti fused-NextN (#590 / bug-858): fuse the N per-step NextN draft decodes into ONE graph on ++// the draft context (ctx_dft, type LLAMA_CONTEXT_TYPE_MTP → LLM_GRAPH_TYPE_DECODER_MTP). This is the ++// throughput fix: the un-fused path pays a host sync per draft step; here one graph emits N greedy ++// drafts with a single sync. Twin of decode_mtp_fused (Gemma assistant) but for the UNIFIED cache — ++// so no mtp_assistant / sched_mtp / kv_iswa. The N fused steps read-only cross-attend the frozen ++// prefix (build_attn_readonly_nextn, Option A); recurrence is carried by the on-device hidden chain. ++// GREEDY-ONLY: no p_min early-stop (the caller reconciles draft length). See ++// docs/features/fused_nextn_mtp.md. Returns 0 on success, negative on error. ++int32_t llama_context::decode_mtp_fused_nextn( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ // NextN self-spec targets are full-attention → unified cache. (iSWA NextN is not a thing; ++ // the Gemma assistant path is decode_mtp_fused.) ++ auto * kv = dynamic_cast(memory.get()); ++ if (!kv) { ++ LLAMA_LOG_ERROR("%s: fused NextN requires a unified llama_kv_cache\n", __func__); ++ return -3; ++ } ++ if (n_steps <= 1) { ++ // Nothing to fuse — the caller should use its per-step path. ++ return -9; ++ } ++ ++ const uint32_t n_embd = model.hparams.n_embd; ++ if (n_embd == 0) { ++ return -4; ++ } ++ ++ // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step ++ // query positions consumed by the wrapper's inp_pos_steps (set_input). ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_embd); ++ data->pos.resize(n_steps); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(seq_id); ++ data->seq_idx[(size_t) seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = (uint32_t) n_steps; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; ++ ++ data->token[0] = last_token; ++ std::memcpy(data->embd.data(), h_prev, n_embd * sizeof(float)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ data->pos[k] = attn_pos + 1 + (llama_pos) k; ++ } ++ ++ // Read-only cross-attention context over the frozen prefix (single mtp_slot_info cell; get_n_kv ++ // reports the full used extent so the mask exposes 0..pmax). NO apply() → no draft-KV write. ++ llama_kv_cache::slot_info_vec_t sinfos; ++ sinfos.push_back(kv->mtp_slot_info(seq_id)); ++ std::vector ubatches; ++ ubatches.push_back(ub); ++ auto mctx = std::make_unique(kv, std::move(sinfos), std::move(ubatches)); ++ if (mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: failed to build mtp_slot_info context\n", __func__); ++ return -5; ++ } ++ ++ // Build/compute the fused N-step graph on the NORMAL sched (graph_params sets n_mtp_steps=N for ++ // DECODER_MTP). Inlined like process_ubatch_mtp — deliberately NOT process_ubatch, whose apply() ++ // would overwrite the pmax prefix cell. ++ mtp_fused_steps = n_steps; ++ ++ // opencoti #590/bug-858: lazily create a DEDICATED scheduler + result for the fused NextN draft so ++ // its ggml graph allocation AND CUDA-graph capture survive across draft rounds. Sharing the main ++ // sched with target decode reset the fused graph's compute buffers every round → 100% rebuild + ++ // recapture AND garbage argmax on reuse. Sized graph_max_nodes*n_steps (the graph unrolls N steps). ++ const int32_t nextn_need = std::max(n_steps, 1); ++ if (!sched_nextn || !gf_res_prev_nextn || nextn_reserved_steps < nextn_need) { ++ sched_nextn.reset(); ++ gf_res_prev_nextn.reset(); ++ const uint32_t n_tok_nextn = std::min(cparams.n_ctx, cparams.n_ubatch); ++ const size_t mn_nextn = (size_t) this->graph_max_nodes(n_tok_nextn) * (size_t) nextn_need; ++ gf_res_prev_nextn.reset(new llm_graph_result(mn_nextn)); ++ sched_nextn.reset(ggml_backend_sched_new( ++ backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), ++ mn_nextn, /*pipeline_parallel*/ false, cparams.op_offload)); ++ if (!sched_nextn) { ++ LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_nextn\n", __func__); ++ gf_res_prev_nextn.reset(); ++ mtp_fused_steps = 1; ++ return -6; ++ } ++ nextn_reserved_steps = nextn_need; ++ } ++ ++ auto * res = gf_res_prev_nextn.get(); ++ llm_graph_params gparams = graph_params(res, ub, mctx.get(), LLM_GRAPH_TYPE_DECODER_MTP); ++ gparams.n_outputs = 1; ++ gparams.sched = sched_nextn.get(); ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ if (!graph_reuse_disable && res->can_reuse(gparams)) { ++ // reuse — dedicated sched_nextn is NOT clobbered by interleaved target decode ++ } else { ++ res->reset(); ++ ggml_backend_sched_reset(sched_nextn.get()); ++ ggml_backend_sched_set_eval_callback(sched_nextn.get(), cparams.cb_eval, cparams.cb_eval_user_data); ++ ++ ggml_cgraph * gf = model.build_graph(gparams); ++ if (!gf) { ++ LLAMA_LOG_ERROR("%s: failed to build fused NextN graph\n", __func__); ++ mtp_fused_steps = 1; ++ return -6; ++ } ++ if (!ggml_backend_sched_alloc_graph(sched_nextn.get(), gf)) { ++ LLAMA_LOG_ERROR("%s: failed to allocate fused NextN graph\n", __func__); ++ mtp_fused_steps = 1; ++ return -6; ++ } ++ } ++ res->set_inputs(&ub); ++ ++ // Compute on the dedicated sched (mirror graph_compute_mtp's CPU-threadpool dance). ++ if (backend_cpu != nullptr) { ++ auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); ++ auto * set_tp_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ++ ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); ++ if (set_tp_fn) { ++ set_tp_fn(backend_cpu, threadpool); ++ } ++ } ++ for (const auto & set_n_threads_fn : set_n_threads_fns) { ++ set_n_threads_fn.second(set_n_threads_fn.first, cparams.n_threads); ++ } ++ status = ggml_backend_sched_graph_compute_async(sched_nextn.get(), res->get_gf()); ++ mtp_fused_steps = 1; ++ if (status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: fused NextN graph compute failed (status %d)\n", __func__, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_nextn.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "fused NextN graph must publish the in-graph argmax block"); ++ GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused NextN argmax must be I32[n_steps]"); ++ ++ std::vector drafts((size_t) n_steps); ++ ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ out_drafts[k] = (llama_token) drafts[(size_t) k]; ++ } ++ ++ if (out_h_prev_last) { ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_embd * sizeof(float)); ++ } ++ ++ return 0; ++} ++ + // opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one + // in-flight request per context (returns -7 if a prior request was not yet waited). + int32_t llama_context::decode_mtp_async( +@@ -3255,6 +3466,10 @@ llm_graph_params llama_context::graph_params( + /*.n_outputs =*/ n_outputs, + /*.cb =*/ graph_get_cb(), + /*.res =*/ res, ++ // opencoti fused-NextN (#590): the DECODER_MTP draft graph fuses N greedy steps when the ++ // fused driver sets mtp_fused_steps=N (reset to 1 after). Every other gtype — and the ++ // per-step AR NextN loop — keeps n_mtp_steps=1 (single-step build_one_step). ++ /*.n_mtp_steps =*/ (gtype == LLM_GRAPH_TYPE_DECODER_MTP ? mtp_fused_steps : 1), + }; + } + +@@ -4330,6 +4545,7 @@ llama_context_params llama_context_default_params() { + /*.type_v =*/ GGML_TYPE_F16, + /*.abort_callback =*/ nullptr, + /*.abort_callback_data =*/ nullptr, ++ /*.ctx_other =*/ nullptr, // opencoti bug-858: dual-context MTP + /*.embeddings =*/ false, + /*.offload_kqv =*/ true, + /*.no_perf =*/ true, +@@ -4415,8 +4631,13 @@ llama_context * llama_init_from_model( + model->hparams.pooling_type, params.pooling_type); + } + ++ // opencoti bug-858: a dual-context gemma4-assistant drafter (ctx_other set) legitimately has ++ // nextn_predict_layers == 0 — its "MTP layers" are the TARGET's backbone K/V, shared in-place ++ // via ctx_other (a KV-less assistant reads the target cells). Only reject a STANDALONE MTP ++ // context (ctx_other == nullptr) that lacks nextn layers; Qwen NextN (layers > 0) is unaffected. + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && +- model->hparams.nextn_predict_layers == 0) { ++ model->hparams.nextn_predict_layers == 0 && ++ params.ctx_other == nullptr) { + LLAMA_LOG_WARN("%s: context type MTP requested but model doesn't contain MTP layers\n", __func__); + return nullptr; + } +@@ -4572,6 +4793,21 @@ float * llama_get_embeddings_pre_norm_ith(llama_context * ctx, int32_t i) { + return ctx->get_embeddings_pre_norm_ith(i); + } + ++// opencoti #590/bug-858: return the on-device backend-sampled draft token (sampling.sampled), produced ++// during the draft decode by the offloaded sampler chain attached via llama_set_sampler (top_k-only → ++// greedy argmax, matching common_sampler_sample). This is the cheap read that replaces the ~152k-vocab ++// common_sampler_sample D2H + CPU softmax/sort (MEASURED 0.98 ms/step). Returns LLAMA_TOKEN_NULL (-1) ++// when backend sampling is inactive → the AR draft loop falls back to the exact CPU sampler. ++llama_token llama_context::get_draft_greedy_ith(int32_t i) { ++ return get_sampled_token_ith(i); ++} ++ ++llama_token llama_get_draft_greedy_ith(llama_context * ctx, int32_t i) { ++ ctx->synchronize(); ++ ++ return ctx->get_draft_greedy_ith(i); ++} ++ + bool llama_set_sampler(llama_context * ctx, llama_seq_id seq_id, llama_sampler * smpl) { + return ctx->set_sampler(seq_id, smpl); + } +@@ -4667,6 +4903,10 @@ llama_memory_t llama_get_memory(const struct llama_context * ctx) { + return ctx->get_memory(); + } + ++llama_context * llama_get_ctx_other(struct llama_context * ctx) { ++ return ctx->get_cparams().ctx_other; // opencoti bug-858: dual-context MTP ++} ++ + void llama_memory_clear(llama_memory_t mem, bool data) { + if (!mem) { + return; +@@ -4955,6 +5195,22 @@ int32_t llama_decode_mtp( + return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); + } + ++int32_t llama_decode_mtp_fused_nextn( ++ llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ if (!ctx) { ++ LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); ++ return -1; ++ } ++ return ctx->decode_mtp_fused_nextn(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_h_prev_last); ++} ++ + // + // perf + // +diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h +index 7f85a13..b6b0efb 100644 +--- a/llama.cpp/src/llama-context.h ++++ b/llama.cpp/src/llama-context.h +@@ -94,6 +94,11 @@ struct llama_context { + float * get_embeddings_pre_norm(); + float * get_embeddings_pre_norm_ith(int32_t i); + ++ // opencoti #590/bug-858: on-device greedy draft token from the previous MTP decode's t_argmax ++ // (returns -1 if the graph published none). Lets the AR spec draft loop skip the full-vocab ++ // common_sampler_sample host round-trip (~0.98 ms/step) when the draft sampler is argmax-greedy. ++ llama_token get_draft_greedy_ith(int32_t i); ++ + llama_token * get_sampled_tokens() const; + llama_token get_sampled_token_ith(int32_t idx); + +@@ -182,6 +187,17 @@ struct llama_context { + llama_token * out_drafts, + float * out_h_prev_last); + ++ // opencoti fused-NextN (#590): fused N-step greedy draft on a DECODER_MTP (Qwen NextN) draft ++ // context using the unified cache. Twin of decode_mtp_fused minus mtp_assistant/sched_mtp/iSWA. ++ int32_t decode_mtp_fused_nextn( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ + // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync + // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams + // no per-step logits); otherwise it submits via decode_mtp_async and blocks in +@@ -430,6 +446,19 @@ private: + ggml_backend_sched_ptr sched_mtp; + llm_graph_result_ptr gf_res_prev_mtp; + ++ // opencoti #590/bug-858: DEDICATED scheduler + result for the fused NextN self-spec draft ++ // (decode_mtp_fused_nextn). It must NOT share the main sched/gf_res_prev with target decode: ++ // every interleaved target decode RESETS the main sched (reallocating compute buffers) and ++ // gf_res_prev, so a fused graph parked there can never satisfy can_reuse AND its tensor memory is ++ // clobbered → 100% rebuild + CUDA-recapture per draft round (measured: build 0.5ms + comp-launch ++ // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched (like the Gemma assistant's ++ // sched_mtp, but ensure_sched_mtp is assistant-only: needs mtp_assistant + iswa cache) isolates the ++ // fused graph so can_reuse compares fused-vs-fused (stable within a 256-cell n_kv bucket) and the ++ // ggml graph + CUDA capture persist across rounds. NextN and Gemma-assistant are mutually exclusive. ++ ggml_backend_sched_ptr sched_nextn; ++ llm_graph_result_ptr gf_res_prev_nextn; ++ int32_t nextn_reserved_steps = 0; ++ + // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for + // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it + // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +index 934adf5..8f0b36b 100644 +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -112,4 +112,9 @@ struct llama_cparams { + + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; ++ ++ // opencoti bug-858: dual-context MTP — the target context whose KV cells the ++ // draft (MTP) context shares IN-PLACE (upstream cparams.ctx_other + mem_other). ++ // nullptr for ordinary contexts. ++ llama_context * ctx_other; + }; +diff --git a/llama.cpp/src/llama-ext.h b/llama.cpp/src/llama-ext.h +index edfa71c..9d57c33 100644 +--- a/llama.cpp/src/llama-ext.h ++++ b/llama.cpp/src/llama-ext.h +@@ -104,3 +104,8 @@ LLAMA_API float * llama_get_embeddings_pre_norm (struct llama_context * ctx); + + // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + LLAMA_API float * llama_get_embeddings_pre_norm_ith(struct llama_context * ctx, int32_t i); ++ ++// opencoti #590/bug-858: on-device greedy draft token from the previous MTP decode's in-graph argmax ++// (t_argmax). Returns the token id, or -1 if none was published. Lets the AR spec draft loop skip the ++// ~0.98 ms/step full-vocab common_sampler_sample (logit D2H + CPU softmax/sort) when p_min==0. ++LLAMA_API llama_token llama_get_draft_greedy_ith(struct llama_context * ctx, int32_t i); +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index e0d5e2a..e0d712e 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -125,14 +125,28 @@ void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { + + // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] + // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. ++ // bug-870: mrope archs (qwen35) size each step's pos tensor I32[4] — fill the M-RoPE ++ // text-token layout [p,p,p,0] (mirrors llm_graph_input_pos::set_input); standard rope = [p]. + for (size_t k = 0; k < inp_pos_steps.size(); ++k) { +- ggml_backend_tensor_set(inp_pos_steps[k], ubatch->pos + k, 0, sizeof(int32_t)); ++ ggml_tensor * pk = inp_pos_steps[k]; ++ const int32_t p = ubatch->pos[k]; ++ if (pk->ne[0] == 4) { ++ const int32_t pos4[4] = { p, p, p, 0 }; ++ ggml_backend_tensor_set(pk, pos4, 0, sizeof(pos4)); ++ continue; ++ } ++ ggml_backend_tensor_set(pk, &p, 0, sizeof(int32_t)); + } + } + + // opencoti F5 M6-S4 mtp + bool llm_graph_input_mtp::can_reuse(const llm_graph_params & params) { +- if (params.gtype != LLM_GRAPH_TYPE_MTP) { ++ // bug-870/#590: the fused NextN driver (decode_mtp_fused_nextn) builds a DECODER_MTP graph, so ++ // gating reuse on MTP-only forced a full N-block MoE graph rebuild+realloc on EVERY draft call — ++ // the per-call build dwarfed the saved host-syncs and made fused SLOWER than the AR loop. Accept ++ // both MTP (Gemma assistant) and DECODER_MTP (Qwen NextN fused); the attn input's own can_reuse ++ // independently vetoes reuse when the prefix mask shape changes, so this stays correct. ++ if (params.gtype != LLM_GRAPH_TYPE_MTP && params.gtype != LLM_GRAPH_TYPE_DECODER_MTP) { + return false; + } + const bool base = inp_last_token && inp_last_token->ne[0] == 1 && inp_h_prev && inp_h_prev->ne[1] == 1; +@@ -607,11 +621,20 @@ bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) { + + bool res = true; + ++ // opencoti bug-858 (2026-07-05): match upstream b9859 — the kq_mask checks must be ++ // INDEPENDENT of the k_idxs blocks. In the KV-less gemma4-assistant draft context the ++ // k/v_idxs are dead nodes (no cache store) and never get buffers, so nesting the mask ++ // check under them skipped it entirely → the boot-time reserve graph (n_kv = full cache ++ // width) was reused for every draft decode → set_input filled only the first n_kv ++ // columns of a full-width mask, leaving stale pool garbage → accept collapse on any ++ // prompt past one KV pad block (gate10: 0.83@114tok → 0.37@354 → 0.00@1314). + // base tensors may not be allocated if there are no non-SWA attention layers + if (self_k_idxs && self_k_idxs->buffer) { + res &= self_k_idxs->ne[0] == params.ubatch.n_tokens; + //res &= self_v_idxs->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there ++ } + ++ if (self_kq_mask && self_kq_mask->buffer) { + res &= can_reuse_kq_mask(self_kq_mask, mctx->get_base(), params.ubatch, params.cparams); + } + +@@ -619,7 +642,9 @@ bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) { + if (self_k_idxs_swa && self_k_idxs_swa->buffer) { + res &= self_k_idxs_swa->ne[0] == params.ubatch.n_tokens; + //res &= self_v_idxs_swa->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there ++ } + ++ if (self_kq_mask_swa && self_kq_mask_swa->buffer) { + res &= can_reuse_kq_mask(self_kq_mask_swa, mctx->get_swa(), params.ubatch, params.cparams); + } + +@@ -2894,7 +2919,32 @@ ggml_tensor * llm_graph_context::build_attn( + // null (build_attn for KV asserts this above at :2102), kq_b too; + // the M2 gate ensures non-transposed V. See docs/features/advanced_kv.md. + ggml_tensor * cur; +- if (mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::GPU_STREAM) { ++ // opencoti-hook: f5-rolling-kv — bug-1840 (#588). POSITION_WINDOW on the FULL-ATTENTION ++ // build_attn overload. The position-window residency tactic was previously wired ONLY into ++ // the iSWA overload (build_attn(llm_graph_input_attn_kv_iswa*), see the mirror ~line 3272); ++ // full-attention models (Qwen etc. — the ONLY class that actually spills, since iSWA KV is ++ // bounded) fell through to the get_k/get_v else-branch below, whose window fallback returns ++ // ggml_concat(window_device, tail_host). That single cross-backend logical tensor makes the ++ // ggml scheduler round-trip the ENTIRE KV cache host<->device EVERY decode step (measured ++ // 12.6 GiB/step both ways -> 0.99 tps at a 4.2 GB tail). Routing spilling layers through ++ // build_attn_mha_position_window keeps the window FA device-resident and streams ONLY the ++ // tail (one H2D copy, no D2H write-back). Guarded to POSITION_WINDOW + FA + no KQ-bias/MLA/ ++ // sinks + an active window, so non-spilling layers are byte-identical. See bug-1840, ++ // docs/features/rolling_kv.md. ++ const bool window_layer = ++ cparams.flash_attn && kq_b == nullptr && v_mla == nullptr && sinks == nullptr && ++ mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::POSITION_WINDOW && ++ mctx_cur->headinfer_window_active(il); ++ if (window_layer) { ++ ggml_tensor * k_win = mctx_cur->get_k_window(ctx0, il); ++ ggml_tensor * v_win = mctx_cur->get_v_window(ctx0, il); ++ ggml_tensor * k_tail = mctx_cur->get_k_tail (ctx0, il); // nullptr when n_kv <= wc ++ ggml_tensor * v_tail = mctx_cur->get_v_tail (ctx0, il); ++ const bool cpu_fa_tail = ++ (k_tail != nullptr) && (q->ne[2] == 1) && mctx_cur->headinfer_tail_is_cpu_fa(); ++ cur = build_attn_mha_position_window(q, k_win, v_win, k_tail, v_tail, ++ kq_b, kq_mask, sinks, v_mla, kq_scale, il, cpu_fa_tail); ++ } else if (mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::GPU_STREAM) { + // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304), S1. GPU_STREAM + // rides the M2 head-split: the device head-group runs a resident FA, + // the host (spilled) head-group runs the streaming-FA op, and the two +@@ -3036,6 +3086,28 @@ ggml_tensor * llm_graph_context::build_attn_mtp( + return cur; + } + ++// opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the frozen prefix ++// KV on the UNIFIED cache. Twin of build_attn_mtp for the non-iSWA llm_graph_input_attn_kv used by ++// Qwen NextN drafters. Reads mctx->get_k/get_v(il) with the input's kq_mask — which exposes the ++// full prefix 0..pmax because get_n_kv reports the cache's used extent, not the mtp_slot_info cell ++// count. Writes NO draft KV; recurrence is carried by the hidden-state chain. wo is applied by the ++// caller (the qwen35* MTP block multiplies by the gate then projects wo). See ++// docs/features/fused_nextn_mtp.md delta #6. ++ggml_tensor * llm_graph_context::build_attn_readonly_nextn( ++ llm_graph_input_attn_kv * inp, ++ ggml_tensor * q_cur, ++ float kq_scale, ++ int il) const { ++ const auto * mctx_cur = inp->mctx; ++ ++ ggml_tensor * kq_mask = inp->get_kq_mask(); ++ ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ ++ return build_attn_mha(q_cur, k, v, nullptr, kq_mask, nullptr, nullptr, kq_scale, il); ++} ++ + static std::unique_ptr build_attn_inp_k_impl( + ggml_context * ctx0, + const llama_ubatch & ubatch, +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +index 0c13245..d3a3499 100644 +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -1041,6 +1041,17 @@ struct llm_graph_context { + int64_t kv_n_head_v, + bool use_k_as_v) const; + ++ // opencoti fused-NextN (Option A): read-only cross-attention into the frozen prefix KV on the ++ // UNIFIED cache. Twin of build_attn_mtp for llm_graph_input_attn_kv (non-iSWA). Reads ++ // mctx->get_k/get_v(il) with the input's kq_mask (which exposes 0..pmax via get_n_kv's used ++ // extent), runs build_attn_mha, writes NO draft KV. wo is applied by the caller. ++ // See docs/features/fused_nextn_mtp.md. ++ ggml_tensor * build_attn_readonly_nextn( ++ llm_graph_input_attn_kv * inp, ++ ggml_tensor * q_cur, ++ float kq_scale, ++ int il) const; ++ + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index 7611271..9628df1 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -33,8 +33,17 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + uint32_t n_pad, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, ++ llama_memory_t mem_other, ++ const layer_share_cb & share, + uint32_t sparse_attn_block_size) : hparams(model.hparams), unified(unified) { + ++ // opencoti bug-858 dual-context MTP — split the source iSWA cache into its base/swa halves ++ // so each inner kv_cache shares with the matching half. GUARD: both stay nullptr when ++ // mem_other == nullptr (every existing caller) ⇒ the inner ctors receive nullptr ⇒ ++ // byte-identical standalone allocation. ++ llama_memory_t mem_other_base = mem_other ? static_cast(mem_other)->get_base() : nullptr; ++ llama_memory_t mem_other_swa = mem_other ? static_cast(mem_other)->get_swa() : nullptr; ++ + // chain filters + const layer_filter_cb filter_base = [&](int32_t il) { + if (filter && !filter(il)) { +@@ -73,7 +82,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + compute_reserve_mib, window_measure_pass, +- n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, sparse_attn_block_size); ++ n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, ++ // opencoti bug-858 dual-context MTP — forward the base-half source + share selector. ++ mem_other_base, share, sparse_attn_block_size); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + +@@ -82,7 +93,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, + headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + compute_reserve_mib, window_measure_pass, +- n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, sparse_attn_block_size); ++ n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, ++ // opencoti bug-858 dual-context MTP — forward the swa-half source + share selector. ++ mem_other_swa, share, sparse_attn_block_size); + } + + void llama_kv_cache_iswa::clear(bool data) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 166b4eb..06fd1ae 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -52,6 +52,13 @@ public: + uint32_t n_pad, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, ++ // opencoti bug-858 dual-context MTP — source iSWA cache; split into ++ // base/swa and forwarded to the two inner kv_base/kv_swa. nullptr => ++ // ordinary standalone cache (byte-identical). Threaded with `share`. ++ llama_memory_t mem_other, ++ // opencoti bug-858 dual-context MTP — per-layer share selector, ++ // forwarded verbatim to both inner caches. Used only when mem_other != null. ++ const layer_share_cb & share, + // opencoti #551 sparse-attn — forwarded to both kv_base and + // kv_swa. Quest block-selector B_SEL; 0 = off (no side-cache). + // See docs/features/sparse_attn.md. +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 13c8c67..4416d38 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -15,6 +15,18 @@ + #include + #include + ++// opencoti bug-858 diagnostic (REMOVE after root-cause): checked map_layer_ids.at() — logs the ++// missing layer index + enclosing function + map size on a miss, instead of an opaque throw. ++template ++static inline int32_t mli_at_checked(const M & m, int32_t il, const char * fn) { ++ auto it = m.find(il); ++ if (it == m.end()) { ++ LLAMA_LOG_ERROR("%s: map_layer_ids MISS il=%d (n_keys=%zu)\n", fn, il, m.size()); ++ throw std::out_of_range("map_layer_ids"); ++ } ++ return it->second; ++} ++ + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + // On Linux, madvise(addr, len, MADV_DONTNEED) is the call that actually + // decommits pages: the kernel zeroes the page table entries, releases the +@@ -184,9 +196,17 @@ static float opencoti_compute_auto_gpu_heads_frac( + size_t free_vram = 0, total_vram = 0; + ggml_backend_dev_memory(dev, &free_vram, &total_vram); + ++ // opencoti bug-1839: --vram-target is a TOTAL-device VRAM cap (rolling_kv.md ++ // Decision 4: "leave headroom for siblings"), NOT a KV-only budget. Weights and ++ // the CUDA context are already resident here, so subtract what the device already ++ // holds (total - free) from the target to get the KV share. free_vram still caps ++ // (we can never make resident more than is actually free right now). + size_t budget = free_vram; + if (vram_target_mib > 0) { +- budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); ++ const size_t vt_bytes = (size_t) vram_target_mib * 1024 * 1024; ++ const size_t already_used = total_vram > free_vram ? total_vram - free_vram : 0; ++ const size_t vt_kv_share = vt_bytes > already_used ? vt_bytes - already_used : 0; ++ budget = std::min(budget, vt_kv_share); + } + // opencoti bug-1342: hold back the MEASURED compute buffer (two-pass) instead of the + // fixed 1536 MiB guess when the context supplied it; 0 → fall back to the default. +@@ -277,16 +297,24 @@ static uint32_t opencoti_compute_resident_window_cells( + + size_t free_vram = 0, total_vram = 0; + ggml_backend_dev_memory(dev, &free_vram, &total_vram); ++ // opencoti bug-1839: --vram-target is a TOTAL-device VRAM cap (rolling_kv.md ++ // Decision 4: "leave headroom for siblings"), NOT a KV-only budget. Weights and ++ // the CUDA context are already resident here, so subtract what the device already ++ // holds (total - free) from the target to get the KV share. free_vram still caps ++ // (we can never make resident more than is actually free right now). + size_t budget = free_vram; + if (vram_target_mib > 0) { +- budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); ++ const size_t vt_bytes = (size_t) vram_target_mib * 1024 * 1024; ++ const size_t already_used = total_vram > free_vram ? total_vram - free_vram : 0; ++ const size_t vt_kv_share = vt_bytes > already_used ? vt_bytes - already_used : 0; ++ budget = std::min(budget, vt_kv_share); + } + budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES + ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES + : 0; + + if (per_cell * (size_t) kv_size <= budget) { +- LLAMA_LOG_INFO("%s: position window = FULLY RESIDENT (KV %.0f MiB fits budget " ++ LLAMA_LOG_WARN("%s: position window = FULLY RESIDENT (KV %.0f MiB fits budget " + "%.0f MiB of %.0f MiB free); no host tail\n", __func__, + per_cell * (size_t) kv_size / 1048576.0, budget / 1048576.0, free_vram / 1048576.0); + return kv_size; +@@ -320,9 +348,10 @@ static uint32_t opencoti_compute_resident_window_cells( + cells = walign; + } + } +- LLAMA_LOG_INFO("%s: position window = %u / %u cells resident (%.0f MiB budget, " +- "%.0f MiB/cell-set); host tail = %u cells\n", __func__, +- cells, kv_size, budget / 1048576.0, per_cell / 1048576.0, kv_size - cells); ++ LLAMA_LOG_WARN("%s: position window = %u / %u cells resident (%.0f MiB budget, " ++ "%.0f MiB/cell-set); host tail = %u cells = %.0f MiB\n", __func__, ++ cells, kv_size, budget / 1048576.0, per_cell / 1048576.0, kv_size - cells, ++ per_cell * (size_t) (kv_size - cells) / 1048576.0); + return cells; + } + +@@ -348,9 +377,32 @@ llama_kv_cache::llama_kv_cache( + llama_swa_type swa_type, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, ++ llama_memory_t mem_other, ++ const layer_share_cb & share, + uint32_t sparse_attn_block_size) : + model(model), hparams(model.hparams), v_trans(v_trans), +- n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type) { ++ n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type), ++ // opencoti bug-858 dual-context MTP — `other` is the TARGET cache (nullptr for every ++ // existing caller). The cells vector is shared behind a shared_ptr: a draft cache reuses ++ // the target's, a standalone cache makes its own. Member-init order is other → v_cells_impl ++ // → v_cells (matches the header declaration order), so v_cells_impl safely reads `other`. ++ other(static_cast(mem_other)), ++ v_cells_impl(other ? other->v_cells_impl : std::make_shared>()), ++ v_cells(*v_cells_impl) { ++ ++ // opencoti bug-858 dual-context MTP — a draft cache (other != nullptr) shares the TARGET's ++ // cells + K/V tensors in place, so its cell count MUST follow the target's allocation (an ++ // oversized view would overflow the source tensors). Mirrors upstream b9859 get_size(). ++ // GUARD: skipped entirely when other == nullptr (every existing caller) ⇒ kv_size is ++ // untouched ⇒ byte-identical to the shipped path. ++ if (other) { ++ const uint32_t size_other = other->get_size(); ++ if (kv_size != size_other) { ++ LLAMA_LOG_WARN("%s: kv_size = %u overridden to %u to match the shared source cache\n", ++ __func__, kv_size, size_other); ++ kv_size = size_other; ++ } ++ } + + GGML_ASSERT(kv_size % n_pad == 0); + +@@ -579,6 +631,37 @@ llama_kv_cache::llama_kv_cache( + continue; + } + ++ // opencoti bug-858 dual-context MTP — share this draft layer's cells + K/V with a ++ // TARGET-cache layer IN PLACE (a KV-less drafter reads the backbone K/V) instead of ++ // allocating its own tensors. Copies the target kv_layer struct (tensor POINTERS + ++ // residency / block-sel metadata; the draft only READS them), retags il to this ++ // cache's model-layer index, and skips allocation (continue). GUARD: `share && other` ++ // — when either is null (every existing caller passes both null) this block is inert, ++ // control falls straight through to the standalone allocation ⇒ byte-identical path. ++ if (share && other) { ++ const int32_t il_share = share(il); ++ ++ if (il_share >= 0) { ++ // opencoti bug-858: match upstream b9859:194 (operator[], NOT .at()) — a missing key ++ // must NOT throw. Diagnose-log if il_share is absent from the target's map (that would ++ // mean our target iSWA inner-cache layer partition differs from upstream's, and []=>0). ++ if (other->map_layer_ids.find(il_share) == other->map_layer_ids.end()) { ++ LLAMA_LOG_WARN("%s: layer %3d: share target %d ABSENT from other map (%zu keys) — []=>0 fallback\n", ++ __func__, il, il_share, other->map_layer_ids.size()); ++ } ++ const auto & layer_share = other->layers[other->map_layer_ids[il_share]]; ++ ++ LLAMA_LOG_WARN("%s: layer %3d: sharing with target layer %d\n", __func__, il, il_share); ++ ++ map_layer_ids[il] = layers.size(); ++ ++ layers.push_back(layer_share); ++ layers.back().il = il; ++ ++ continue; ++ } ++ } ++ + if (n_embd_head_k_all == 0) { + n_embd_head_k_all = (int32_t) hparams.n_embd_head_k(il); + } else if (n_embd_head_k_all > 0 && n_embd_head_k_all != (int32_t) hparams.n_embd_head_k(il)) { +@@ -754,6 +837,21 @@ llama_kv_cache::llama_kv_cache( + if (!ctx_cpu_s) { + throw std::runtime_error("failed to create CPU ggml context for headinfer kv cache"); + } ++ // opencoti bug-1342 diag: the window-tail host K/V is streamed device-ward ++ // per decode tile via strided cudaMemcpy2DAsync. PINNED host memory makes ++ // that async + full-PCIe; a pageable 'CPU' buft makes it SYNCHRONOUS and ++ // slow (per-row). Log the resolved buft ONCE so a boot log settles ++ // pinned-vs-pageable without a debugger (WARN so -lv 3 keeps it). ++ if (layer_window) { ++ static bool opencoti_tail_buft_logged = false; ++ if (!opencoti_tail_buft_logged) { ++ opencoti_tail_buft_logged = true; ++ LLAMA_LOG_WARN("%s: window-tail host buffer buft = '%s' (pinned host = " ++ "async full-PCIe DMA; plain 'CPU' = pageable → SYNCHRONOUS slow " ++ "strided H2D per decode tile)\n", __func__, ++ ggml_backend_buft_name(cpu_buft)); ++ } ++ } + // window mode: ALL heads, tail_c cells; M2: CPU head subset, kv_size cells. + const uint32_t host_k_embd = layer_window ? n_embd_k_gqa : n_embd_k_cpu; + const uint32_t host_v_embd = layer_window ? n_embd_v_gqa : n_embd_v_cpu; +@@ -1014,6 +1112,14 @@ llama_kv_cache::llama_kv_cache( + + // opencoti F4 M3 Phase 1+2 — see docs/decisions/0001-lazy-slot-context.md + void llama_kv_cache::ensure_cleared(uint32_t up_to_cells) { ++ // opencoti bug-858 (bug-2102): a shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // ALIASES the target's k/v tensors (A3 layer_share) and starts with n_cells_cleared == 0 — ++ // running the lazy clear here MEMSETS the target's LIVE KV (the drafter's first decode zeroed ++ // the head of the target's prompt keys → 4-17-logit trajectory flips scaling with prompt ++ // length). The shared cache is strictly READ-ONLY [TAG_KV_CACHE_SHARE_CELLS]: never clear. ++ if (other) { ++ return; ++ } + // up_to_cells == 0 means "clear up to the current soft cap". Callers + // that need full coverage (state_read) pass kv_size_max. + const uint32_t target = up_to_cells == 0 +@@ -1189,6 +1295,11 @@ static void opencoti_decommit_layer_range(ggml_tensor * t, + } + + void llama_kv_cache::shrink_if_idle() { ++ // opencoti bug-858 (bug-2102): shared draft cache (other!=null) aliases the TARGET's k/v ++ // tensors — shrinking here would mutate the target's live cache. READ-ONLY: never shrink. ++ if (other) { ++ return; ++ } + if (slot_shrink_idle_ms_ == 0) { + return; // shrink-on-idle disabled + } +@@ -1296,6 +1407,12 @@ void llama_kv_cache::clear(bool data) { + } + + bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return true; ++ } + GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size())); + + if (p0 < 0) { +@@ -1490,6 +1607,12 @@ std::vector llama_kv_cache::seq_key_scores(llama_seq_id seq_id, int32_t l + } + + void llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_ASSERT(seq_id_src >= 0 && (size_t) seq_id_src < seq_to_stream.size()); + GGML_ASSERT(seq_id_dst >= 0 && (size_t) seq_id_dst < seq_to_stream.size()); + +@@ -1577,6 +1700,12 @@ void llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, ll + } + + void llama_kv_cache::seq_keep(llama_seq_id seq_id) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + + auto & cells = v_cells[seq_to_stream[seq_id]]; +@@ -1599,6 +1728,12 @@ void llama_kv_cache::seq_keep(llama_seq_id seq_id) { + } + + void llama_kv_cache::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + GGML_ASSERT(hparams.n_pos_per_embd() == 1 && "seq_add() is only supported for n_pos_per_embd() == 1"); + +@@ -1644,6 +1779,12 @@ void llama_kv_cache::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, ll + } + + void llama_kv_cache::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + GGML_ASSERT(hparams.n_pos_per_embd() == 1 && "seq_div() is only supported for n_pos_per_embd() == 1"); + +@@ -1678,6 +1819,12 @@ void llama_kv_cache::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, in + } + + llama_pos llama_kv_cache::seq_pos_min(llama_seq_id seq_id) const { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return other->seq_pos_min(seq_id); ++ } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + + const auto & cells = v_cells[seq_to_stream[seq_id]]; +@@ -1686,6 +1833,12 @@ llama_pos llama_kv_cache::seq_pos_min(llama_seq_id seq_id) const { + } + + llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return other->seq_pos_max(seq_id); ++ } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + + const auto & cells = v_cells[seq_to_stream[seq_id]]; +@@ -1865,6 +2018,12 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vectorget_sched(); +@@ -2188,6 +2347,12 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, + } + + void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + // opencoti F4 M3 Phase 3 — see docs/decisions/0001-lazy-slot-context.md + // Refresh the idle timestamp on every actual write. shrink_if_idle uses + // this to gate the shrink path. +@@ -2340,7 +2505,7 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { + } + + ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + + const auto & layer = layers[ikv]; + +@@ -2462,7 +2627,7 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k + } + + ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + + const auto & layer = layers[ikv]; + +@@ -2923,7 +3088,7 @@ uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const { + } + + ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split"); + +@@ -2959,7 +3124,7 @@ ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t + // views it in-place. n_kv is irrelevant — kbounds spans all selectable blocks. + ggml_tensor * llama_kv_cache::get_kbounds(ggml_context * ctx, int32_t il, const slot_info & sinfo) const { + GGML_UNUSED(ctx); +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + if (layer.kbounds_per_stream.empty()) { + return nullptr; +@@ -2973,7 +3138,7 @@ ggml_tensor * llama_kv_cache::get_kbounds(ggml_context * ctx, int32_t il, const + // re-writes it only every cparams.sparse_attn_refresh decode tokens; FA-VEC reads it every step. + ggml_tensor * llama_kv_cache::get_block_sel(ggml_context * ctx, int32_t il, const slot_info & sinfo) const { + GGML_UNUSED(ctx); +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + if (layer.block_sel_per_stream.empty()) { + return nullptr; +@@ -2982,7 +3147,7 @@ ggml_tensor * llama_kv_cache::get_block_sel(ggml_context * ctx, int32_t il, cons + } + + ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 + && !layer.k_cpu_per_stream.empty() +@@ -2998,6 +3163,18 @@ ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t + + auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { + ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ // opencoti #586 diag: report the ACTUAL allocated buffer buft (the boot WARN ++ // reports the *intended* cpu_buft, masking a silent cudaMallocHost fallback to ++ // pageable CPU — the H2D tail stream then runs at ~half PCIe / non-overlapping). ++ { ++ static bool opencoti_actual_tail_buft_logged = false; ++ if (!opencoti_actual_tail_buft_logged) { ++ opencoti_actual_tail_buft_logged = true; ++ LLAMA_LOG_WARN("%s: ACTUAL tail: data=%p buffer=%p buft='%s'\n", __func__, ++ (void *) kc_s->data, (void *) kc_s->buffer, ++ kc_s->buffer ? ggml_backend_buft_name(ggml_backend_buffer_get_type(kc_s->buffer)) : "NULL-BUFFER"); ++ } ++ } + return ggml_view_4d(ctx, kc_s, + n_embd_head_k, n_head_cpu, n_kv, 1, + ggml_row_size(kc_s->type, n_embd_head_k), +@@ -3014,7 +3191,7 @@ ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t + } + + ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split"); + // M2 requires non-transposed V (FA on). The split-active branch in +@@ -3045,7 +3222,7 @@ ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t + } + + ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.gpu_heads > 0 + && !layer.v_cpu_per_stream.empty() +@@ -3085,7 +3262,7 @@ ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t + // ne[1] == window_cells, so its dim-3 (stream/position) stride is computed off + // k_s->ne[1] (the audit point — NOT kv_size); the host tail likewise off kc_s->ne[1]. + ggml_tensor * llama_kv_cache::get_k_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.window_cells > 0 && "get_k_window called without an active position window"); + +@@ -3113,7 +3290,7 @@ ggml_tensor * llama_kv_cache::get_k_window(ggml_context * ctx, int32_t il, uint3 + } + + ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.window_cells > 0 + && !layer.k_cpu_per_stream.empty() +@@ -3125,12 +3302,35 @@ ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_ + const uint64_t n_embd_k_gqa2 = (uint64_t) n_head_kv * n_embd_head_k; + const uint32_t wc = layer.window_cells; + const uint32_t n_tail = n_kv > wc ? n_kv - wc : 0; ++ { ++ static bool opencoti_gkt_entry_logged = false; ++ if (!opencoti_gkt_entry_logged) { ++ opencoti_gkt_entry_logged = true; ++ ggml_tensor * kc0 = layer.k_cpu_per_stream.empty() ? nullptr : layer.k_cpu_per_stream[0]; ++ LLAMA_LOG_WARN("%s: ENTRY il=%d n_kv=%u wc=%u n_tail=%u kc0=%p kc0.buffer=%p buft=%s\n", ++ __func__, il, n_kv, wc, n_tail, (void *)(kc0 ? kc0->data : nullptr), ++ (void *)(kc0 ? kc0->buffer : nullptr), ++ (kc0 && kc0->buffer) ? ggml_backend_buft_name(ggml_backend_buffer_get_type(kc0->buffer)) : "NULL"); ++ } ++ } + if (n_tail == 0) { + return nullptr; // no overflow → fully resident window, single-region FA + } + + auto build_stream_view = [&](uint32_t s_cache) -> ggml_tensor * { + ggml_tensor * kc_s = layer.k_cpu_per_stream[s_cache]; ++ // opencoti #586 diag: report the ACTUAL allocated buffer buft (the boot WARN ++ // reports the *intended* cpu_buft, masking a silent cudaMallocHost fallback to ++ // pageable CPU — the H2D tail stream then runs at ~half PCIe / non-overlapping). ++ { ++ static bool opencoti_actual_tail_buft_logged = false; ++ if (!opencoti_actual_tail_buft_logged) { ++ opencoti_actual_tail_buft_logged = true; ++ LLAMA_LOG_WARN("%s: ACTUAL tail: data=%p buffer=%p buft='%s'\n", __func__, ++ (void *) kc_s->data, (void *) kc_s->buffer, ++ kc_s->buffer ? ggml_backend_buft_name(ggml_backend_buffer_get_type(kc_s->buffer)) : "NULL-BUFFER"); ++ } ++ } + return ggml_view_4d(ctx, kc_s, + n_embd_head_k, n_head_kv, n_tail, 1, + ggml_row_size(kc_s->type, n_embd_head_k), +@@ -3147,7 +3347,7 @@ ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_ + } + + ggml_tensor * llama_kv_cache::get_v_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.window_cells > 0 && "get_v_window called without an active position window"); + GGML_ASSERT(!v_trans && "position window requires non-transposed V"); +@@ -3176,7 +3376,7 @@ ggml_tensor * llama_kv_cache::get_v_window(ggml_context * ctx, int32_t il, uint3 + } + + ggml_tensor * llama_kv_cache::get_v_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; + GGML_ASSERT(layer.window_cells > 0 + && !layer.v_cpu_per_stream.empty() +@@ -3230,7 +3430,7 @@ uint32_t llama_kv_cache::headinfer_window_cells(int32_t il) const { + } + + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + + const auto & layer = layers[ikv]; + +@@ -3413,7 +3613,7 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + } + + ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const { +- const int32_t ikv = map_layer_ids.at(il); ++ const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + + const auto & layer = layers[ikv]; + +@@ -4365,6 +4565,10 @@ void llm_graph_input_k_shift::set_input(const llama_ubatch * ubatch) { + } + + ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_context * lctx) const { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ GGML_ASSERT(!other && "shift on a shared draft cache is a bug"); + auto * ctx = res->get_ctx(); + auto * gf = res->get_gf(); + +@@ -4467,6 +4671,12 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co + } + + void llama_kv_cache::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_UNUSED(flags); + + io.write(&n_stream, sizeof(n_stream)); +@@ -4520,6 +4730,12 @@ void llama_kv_cache::state_write(llama_io_write_i & io, llama_seq_id seq_id, lla + } + + void llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { ++ // opencoti bug-858 (bug-2097): shared draft cache (gemma4-assistant ctx_dft, other!=null) ++ // is READ-ONLY over the target's cells — mirror upstream b9859 [TAG_KV_CACHE_SHARE_CELLS]. ++ // Inert on the target/standalone path (other==nullptr). ++ if (other) { ++ return; ++ } + GGML_UNUSED(flags); + + GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size())); +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +index 924f7fe..7b3afc3 100644 +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -146,6 +146,14 @@ public: + llama_swa_type swa_type, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, ++ // opencoti bug-858 dual-context MTP — source cache whose cells + ++ // per-layer K/V tensors the draft cache shares IN PLACE. nullptr => ++ // ordinary standalone cache (byte-identical to pre-bug-858). Threaded ++ // with `share`; both are null for every target / existing caller. ++ llama_memory_t mem_other, ++ // opencoti bug-858 dual-context MTP — per-layer share selector; see ++ // llama_memory_i::layer_share_cb. Consulted only when mem_other != null. ++ const layer_share_cb & share, + // opencoti #551 sparse-attn — Quest block-selector KV-block + // size B_SEL (docs/features/sparse_attn.md). 0 = sparse-attn + // OFF: no kmin/kmax side-cache allocated (byte-identical). +@@ -548,7 +556,20 @@ private: + // note: this is not part of the KV state and it's only used to speed-up the find_slot() method + std::vector v_heads; + +- std::vector v_cells; ++ // opencoti bug-858 dual-context MTP — the TARGET cache this draft cache shares its ++ // cells + layer tensors with, or nullptr for an ordinary standalone cache. Set once ++ // at construction from the ctor's mem_other. Every existing caller passes nullptr, so ++ // `other == nullptr` is the shipped, byte-identical path. Declared BEFORE v_cells_impl ++ // so the init-list (which reads `other`) runs in the correct member order. ++ llama_kv_cache * other = nullptr; ++ ++ // opencoti bug-858 dual-context MTP — the cells vector is held behind a shared_ptr so a ++ // draft cache (other != nullptr) can SHARE the target's cells in place. `v_cells` is a ++ // reference to *v_cells_impl, so EVERY existing v_cells[...] / v_cells.resize(...) access ++ // is unchanged. When other == nullptr this owns a freshly-made vector — identical layout ++ // and lifetime to the previous plain `std::vector v_cells;` member. ++ std::shared_ptr> v_cells_impl; ++ std::vector & v_cells; + + // maps from a sequence id to a stream id + std::vector seq_to_stream; +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +index 7301ad5..3cf8e4a 100644 +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -65,7 +65,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + filter_attn == nullptr ? + [&](int32_t il) { return !hparams.is_recurrent(il); } + : filter_attn, +- nullptr ++ nullptr, ++ // opencoti bug-858 dual-context MTP — hybrid-iswa attn cache is standalone: no sharing. ++ /* mem_other */ nullptr, ++ /* share */ nullptr + )), + mem_recr(new llama_memory_recurrent( + model, +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +index 86e4134..0fa114d 100644 +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -65,7 +65,10 @@ llama_memory_hybrid::llama_memory_hybrid( + filter_attn == nullptr ? + [&](int32_t il) { return !hparams.is_recurrent(il); } + : filter_attn, +- nullptr ++ nullptr, ++ // opencoti bug-858 dual-context MTP — hybrid attn cache is standalone: no sharing. ++ /* mem_other */ nullptr, ++ /* share */ nullptr + )), + mem_recr(new llama_memory_recurrent( + model, +diff --git a/llama.cpp/src/llama-memory.h b/llama.cpp/src/llama-memory.h +index 3e79826..98e4f26 100644 +--- a/llama.cpp/src/llama-memory.h ++++ b/llama.cpp/src/llama-memory.h +@@ -24,6 +24,10 @@ struct llama_memory_params { + bool swa_full; + + llama_context_type ctx_type; ++ ++ // opencoti bug-858: dual-context MTP — source memory whose cells the draft cache ++ // shares in-place (upstream mem_other). nullptr => ordinary standalone cache. ++ llama_memory_t mem_other; + }; + + enum llama_memory_status { +@@ -77,6 +81,13 @@ struct llama_memory_i { + // return negative value to indicate that the layer il should not reuse memory + using layer_reuse_cb = std::function; + ++ // opencoti bug-858 dual-context MTP — maps a draft-cache layer il to the TARGET ++ // cache's layer whose cells + K/V tensors it shares IN PLACE (a KV-less drafter reads ++ // the backbone K/V). Return negative to indicate layer il does NOT share (allocate ++ // normally). Consulted ONLY when the ctor is given a non-null mem_other; a nullptr ++ // share (every existing caller) means no sharing ⇒ byte-identical standalone cache. ++ using layer_share_cb = std::function; ++ + virtual ~llama_memory_i() = default; + + // split the input batch into a set of ubatches and verify that they can fit into the cache +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +index 3df76f7..d7804be 100644 +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2079,6 +2079,24 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + GGML_ASSERT(hparams.is_swa_any()); + ++ // opencoti bug-858 dual-context MTP (A3) — when THIS model is the ++ // gemma4-assistant drafter created as ctx_dft (cparams.ctx_other = the ++ // target context), share the TARGET's KV cells IN PLACE: the assistant has ++ // no K/V projections, so each of its attention layers reads the target ++ // backbone's last full (n_layer-1) / SWA (n_layer-2) layer. Mirrors upstream ++ // b9859 create_memory LLM_ARCH_GEMMA4_ASSISTANT. Inert for every other arch ++ // and when ctx_other is null → nullptr/nullptr → byte-identical standalone cache. ++ llama_memory_t mtp_mem_other = nullptr; ++ llama_kv_cache::layer_share_cb mtp_share = nullptr; ++ if (arch == LLM_ARCH_GEMMA4_ASSISTANT && cparams.ctx_other != nullptr) { ++ mtp_mem_other = llama_get_memory(cparams.ctx_other); ++ mtp_share = [&](int32_t il) { ++ const llama_model * model_other = llama_get_model(cparams.ctx_other); ++ const int32_t n_layer_other = (int32_t) llama_model_n_layer(model_other); ++ return hparams.is_swa(il) ? (n_layer_other - 2) : (n_layer_other - 1); ++ }; ++ } ++ + res = new llama_kv_cache_iswa( + *this, + params.type_k, +@@ -2109,6 +2127,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + 1, + filter, + reuse, ++ // opencoti bug-858 dual-context MTP (A3) — share the target's KV when ++ // this is the gemma4-assistant ctx_dft (mtp_mem_other/mtp_share set ++ // above); nullptr/nullptr for the target + every other arch → identical. ++ /* mem_other */ mtp_mem_other, ++ /* share */ mtp_share, + // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) + cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); + } else { +@@ -2144,6 +2167,10 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + hparams.swa_type, + filter, + nullptr, ++ // opencoti bug-858 dual-context MTP — target / standalone cache: ++ // no sharing in A2 (A3 wires the draft cache with a non-null source). ++ /* mem_other */ nullptr, ++ /* share */ nullptr, + // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) + cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); + } +diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp +index 51f935c..cab4fa8 100644 +--- a/llama.cpp/src/models/gemma4-assistant.cpp ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -382,11 +382,17 @@ void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); + +- uint32_t n_kv_shared_layers = 0; +- ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); +- +- hparams.n_layer_kv_from_start = hparams.n_layer - (int32_t) n_kv_shared_layers; +- hparams.f_attention_scale = 1.0f; ++ // opencoti bug-858 dual-context MTP — the gemma4-assistant is a KV-LESS drafter: its attention ++ // layers do NOT own K/V. When run as ctx_dft they alias the TARGET's backbone K/V in place via ++ // mem_other/share (llama-model.cpp create_memory A3 + llama-kv-cache.cpp share branch). So ++ // n_layer_kv_from_start MUST stay -1 (has_kv() default => true for every layer), matching ++ // upstream b9859 which does NOT set it for LLM_ARCH_GEMMA4_ASSISTANT. Deriving it from ++ // ATTENTION_SHARED_KV_LAYERS (= n_layer for the drafter) yields 0 → has_kv() false for all ++ // layers → the KV-cache ctor `continue`s past every layer BEFORE the share branch → empty ++ // map_layer_ids → "get_k: map_layer_ids MISS il=0 (n_keys=0)" boot crash. That GGUF key drives ++ // the TARGET's intra-model elastic KV reuse (#417), a different mechanism from the cross-model ++ // share, and is inert for the KV-less drafter. ++ hparams.f_attention_scale = 1.0f; + + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); +@@ -434,7 +440,16 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + + // opencoti F5 M6-S4 mtp: GGUF tensor names upstream-aligned (#23398/#24282) — nextn.* / masked_embd_*. + // The C++ member names stay mtp_* (control-plane; our single-context engine, not upstream's ctx_other). +- mtp_pre_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_PRE, "weight"), {2 * (int64_t) n_bb, n_embd}, 0); ++ // ++ // opencoti bug-858 (dual-context MTP): pre_projection is classified LLM_TENSOR_LAYER_INPUT, so the ++ // plain create_tensor lands it on the CPU buft. The single-context facade tolerates that (it runs on ++ // the target's sched_mtp, which has the CPU backend registered), but a REAL draft context ctx_dft ++ // (llama_init_from_model over the nested assistant) registers only the assistant's *offloaded* (CUDA) ++ // weight buffers — a CPU-resident weight then reads back uninitialized → the whole drafter forward is ++ // NaN → 0% accept. Force it onto the output-head (GPU-when-offloaded) buft, exactly like tok_embd ++ // (#408): device-config-respecting (stays CPU for a CPU-only drafter) and placement-only, so the ++ // facade path stays byte-identical. post_projection/output_norm are LAYER_OUTPUT → already dev_output. ++ mtp_pre_projection = create_tensor_output_head(tn(LLM_TENSOR_NEXTN_PROJ_PRE, "weight"), {2 * (int64_t) n_bb, n_embd}, 0); + mtp_post_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_POST, "weight"), {n_embd, (int64_t) n_bb}, 0); + + if (hparams.use_ordered_embeddings) { +@@ -481,8 +496,167 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + } + } + +-std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params &) const { +- throw std::runtime_error( +- "gemma4_assistant cannot be used as a primary model (-m). " +- "Load the Gemma 4 target with -m, then call llama_model_load_mtp_from_file() with the assistant GGUF."); ++// opencoti bug-858 dual-context MTP (A6): the gemma4-assistant graph built when the assistant is ++// created as its OWN llama_context (ctx_dft) with cparams.ctx_other = the target context. Mirrors ++// upstream b9859 llama_model_gemma4_assistant::graph. The assistant is KV-LESS (wq/wo only, no ++// wk/wv): each attention layer reads the TARGET's shared K/V *in place* via the standard iSWA ++// build_attn (Qcur only, null k_cur/v_cur -> no store). The draft context's create_memory (A3) ++// shares the target's KV cells and aliases the target's last full (n_layer-1) / SWA (n_layer-2) ++// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V — ++// which is exactly what the single-context path did explicitly via build_attn_mtp(..., il_kv). ++// The input token embedding + backbone hidden come from the target model via ctx_other. ++// ++// This is DISTINCT from the single-context llm_build_gemma4_mtp (built by the gemma4 TARGET's ++// build_arch_graph, gemma4.cpp) which is kept fully intact as the fallback drafter. ++// ++// Faithful to upstream b9859 (and unlike our single-context build): dense LM head over tok_embd ++// (no ordered-embeddings/centroid path — b9859 loads centroids but ignores them in the graph), ++// no final-logit softcapping (monotonic -> argmax-invariant), and no control-vector application ++// (inert without a control vector). See docs/evaluations/mtp.md. ++struct llm_build_gemma4_assistant_dual : public llm_graph_context { ++ llm_build_gemma4_assistant_dual(const llama_model & model, const llm_graph_params & params) ++ : llm_graph_context(params) { ++ const int64_t n_bb = hparams.n_embd_out_impl; // backbone width (== target n_embd) ++ GGML_ASSERT(n_bb > 0); ++ GGML_ASSERT(model.mtp_pre_projection != nullptr && model.mtp_post_projection != nullptr); ++ ++ GGML_ASSERT(cparams.ctx_other != nullptr && ++ "gemma4_assistant is a KV-less drafter: it can only run as an MTP draft context " ++ "(ctx_dft) created with cparams.ctx_other = the Gemma 4 target context."); ++ const llama_model * model_other = llama_get_model(cparams.ctx_other); ++ ++ // Draft inputs: the drafted token ids + the paired target backbone hidden (h_prev). ++ ggml_tensor * inp_tokens; ++ ggml_tensor * inp_h; ++ { ++ auto inp = std::make_unique(n_bb); ++ ++ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens); ++ cb(inp->tokens, "inp_tokens", -1); ++ ggml_set_input(inp->tokens); ++ inp_tokens = inp->tokens; ++ res->t_inp_tokens = inp->tokens; ++ ++ inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, ubatch.n_tokens); ++ cb(inp->embd, "inp_h", -1); ++ ggml_set_input(inp->embd); ++ inp_h = inp->embd; ++ res->t_inp_embd = inp->embd; ++ ++ res->add_input(std::move(inp)); ++ } ++ ++ // Read the TARGET's token embedding through ctx_other (the assistant reuses the target's ++ // tok_embd for the input pipeline), scaled by sqrt(n_bb) like Gemma 4's input pipeline. ++ ggml_tensor * x = ggml_get_rows(ctx0, model_other->tok_embd, inp_tokens); ++ x = ggml_scale(ctx0, x, sqrtf((float) n_bb)); ++ cb(x, "inp_embd_target", -1); ++ ++ ggml_tensor * xh = ggml_concat(ctx0, x, inp_h, 0); ++ cb(xh, "inp_xh", -1); ++ ++ ggml_tensor * cur = build_lora_mm(model.mtp_pre_projection, xh); ++ cb(cur, "pre_proj", -1); ++ ++ auto * inp_attn = build_attn_inp_kv_iswa(); ++ ggml_tensor * inp_pos = build_inp_pos(); ++ ggml_tensor * inp_out_ids = build_inp_out_ids(); ++ ++ ggml_tensor * inpL = cur; ++ ++ for (int il = 0; il < n_layer; ++il) { ++ const bool is_swa = hparams.is_swa(il); ++ ++ const int64_t n_embd_head = hparams.n_embd_head_k(il); ++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); ++ const int64_t n_head = hparams.n_head(il); ++ ++ const float freq_base_l = model.get_rope_freq_base(cparams, il); ++ const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ++ const int n_rot_l = hparams.n_rot(il); ++ ++ ggml_tensor * cur_norm = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur_norm, "attn_norm", il); ++ ++ ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur_norm); ++ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); ++ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); ++ cb(Qcur, "Qcur_normed", il); ++ ++ ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs; ++ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, ++ freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); ++ cb(Qcur, "Qcur_pos", il); ++ ++ // KV-less cross-attention into the shared/aliased target K/V (null k_cur/v_cur -> no store). ++ cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr, ++ Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); ++ ++ if (il == n_layer - 1 && inp_out_ids) { ++ cur = ggml_get_rows(ctx0, cur, inp_out_ids); ++ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); ++ } ++ ++ cur = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "attn_post_norm", il); ++ ++ ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); ++ cb(attn_out, "attn_out", il); ++ ++ GGML_ASSERT(model.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); ++ ++ cur = build_norm(attn_out, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "ffn_norm", il); ++ ++ cur = build_ffn(cur, ++ model.layers[il].ffn_up, nullptr, nullptr, ++ model.layers[il].ffn_gate, nullptr, nullptr, ++ model.layers[il].ffn_down, nullptr, nullptr, ++ nullptr, ++ LLM_FFN_GELU, LLM_FFN_PAR, il); ++ cb(cur, "ffn_out", il); ++ ++ cur = build_norm(cur, model.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "ffn_post_norm", il); ++ ++ cur = ggml_add(ctx0, cur, attn_out); ++ ++ if (model.layers[il].out_scale) { ++ cur = ggml_mul(ctx0, cur, model.layers[il].out_scale); ++ cb(cur, "out_scaled", il); ++ } ++ ++ inpL = cur; ++ } ++ ++ cur = inpL; ++ ++ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "result_norm", -1); ++ ++ // LM head (dense, tied to tok_embd — ours' create_tensor_output_head; upstream uses a ++ // DUPLICATED `output`, same weight). Full-vocab logits for greedy verify/draft. ++ ggml_tensor * logits = build_lora_mm(model.tok_embd, cur); ++ cb(logits, "result_output", -1); ++ res->t_logits = logits; ++ ++ // Recurrent backbone hidden for the NEXT draft step (post-projected to n_bb), exposed via ++ // t_h_pre_norm so the draft loop harvests it with llama_get_embeddings_pre_norm_ith(ctx_dft). ++ // Mirrors upstream res->t_h_nextn = mul_mat(nextn_proj_post, cur). ++ ggml_tensor * h_next = build_lora_mm(model.mtp_post_projection, cur); ++ cb(h_next, "h_pre_norm", -1); ++ res->t_h_pre_norm = h_next; ++ ++ ggml_build_forward_expand(gf, logits); ++ ggml_build_forward_expand(gf, h_next); ++ } ++}; ++ ++std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { ++ // opencoti bug-858 dual-context MTP (A6): build the gemma4-assistant drafter graph. This model ++ // is KV-less and only valid as an MTP draft context (ctx_dft) created FROM it with ++ // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. The single-context ++ // engine (llm_build_gemma4_mtp) is built by the gemma4 TARGET's build_arch_graph instead and ++ // never reaches here; it remains the fallback drafter. ++ return std::make_unique(*this, params); + } +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +index a7b1598..bdf5990 100644 +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -171,6 +171,15 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + ++ // opencoti-hook: gemma4-mtp-hidden (bug-858 dual-context MTP) — when a context extracts the ++ // pre-norm hidden for ALL positions (embeddings_pre_norm && !masked, i.e. the MTP *target* whose ++ // shifted per-position hidden seeds the drafter), DEFER the inp_out_ids row-strip until after ++ // t_h_pre_norm is captured, so t_h_pre_norm carries all n_tokens rows. Upstream b9859 gates the ++ // in-loop strip on embeddings_nextn_masked and strips late when !masked (gemma4.cpp:278/416). ++ // Normal decode (embeddings_pre_norm==false) and the masked draft keep ours' early strip → ++ // byte-identical. See docs/evaluations/mtp.md and UPSTREAM_SYNC.md. ++ const bool mtp_defer_out_ids = cparams.embeddings_pre_norm && !cparams.embeddings_pre_norm_masked; ++ + ggml_tensor * inp_per_layer = nullptr; + if (model.per_layer_tok_embd) { + inp_per_layer = build_inp_per_layer(); +@@ -253,7 +262,9 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + } + + // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing +- if (il == n_layer - 1 && inp_out_ids) { ++ // opencoti bug-858: skip the early strip for the MTP target (mtp_defer_out_ids) so the last ++ // layer runs on all n_tokens rows and t_h_pre_norm below stays ungathered. ++ if (il == n_layer - 1 && inp_out_ids && !mtp_defer_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } +@@ -353,7 +364,9 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] + + // TODO @ngxson : improve this +- if (il == n_layer - 1 && inp_out_ids) { ++ // opencoti bug-858: same deferred strip as the main path (keep per-layer input full for ++ // the MTP target so its last-layer residual matches the ungathered hidden). ++ if (il == n_layer - 1 && inp_out_ids && !mtp_defer_out_ids) { + inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids); + } + +@@ -384,6 +397,22 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + model.output_norm, nullptr, + LLM_NORM_RMS, -1); + ++ // opencoti-hook: gemma4-mtp-hidden (bug-858 dual-context MTP) — expose the POST-output-norm ++ // hidden state as t_h_pre_norm so a gemma4-assistant draft context (ctx_dft with ctx_other=this) ++ // can read it via llama_get_embeddings_pre_norm_ith() as the recurrent h input. Mirrors upstream ++ // b9859 gemma4.cpp res->t_h_nextn = cur (post-final-norm, captured PRE row-strip). For the MTP ++ // target (mtp_defer_out_ids) this tensor carries all n_tokens rows so the driver can shift the ++ // per-position hidden; for normal decode / masked draft the rows were already stripped in the ++ // last layer, so it equals the n_outputs t_embd below. See docs/evaluations/mtp.md. ++ cb(cur, "h_pre_norm", -1); ++ res->t_h_pre_norm = cur; ++ ++ // opencoti bug-858: deferred row-strip — t_h_pre_norm captured all positions above; now strip to ++ // the output rows for the logits / t_embd path (upstream gemma4.cpp:416 `if (!masked) get_rows`). ++ if (mtp_defer_out_ids && inp_out_ids) { ++ cur = ggml_get_rows(ctx0, cur, inp_out_ids); ++ } ++ + cb(cur, "result_norm", -1); + res->t_embd = cur; + +diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp +index 131f1fa..3a21d69 100644 +--- a/llama.cpp/src/models/qwen35.cpp ++++ b/llama.cpp/src/models/qwen35.cpp +@@ -215,6 +215,20 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + } + cur = inpL; + ++ // opencoti-hook: qwen-nextn-mtp-hidden (bug-858) — see docs/evaluations/mtp.md. ++ // Upstream (b9859) exposes the MTP/NextN hidden as the POST-output-norm state (t_h_nextn); ++ // we mirror that here (apply output_norm FIRST, then expose it) so t_h_pre_norm points at the ++ // same tensor upstream's nextn.hnorm consumes — a syncability alignment, not a perf fix. ++ // MEASURED: on Qwen3.6-35B-A3B-MTP this reorder is numerically INERT (output_norm ≈ identity, ++ // so RMSNorm-then-hnorm == hnorm; draft acceptance is byte-identical 0.626 either way on GPU). ++ // The real acceptance gap vs upstream (ours 0.626 vs 0.770) is NOT the hidden-capture point — ++ // it is localized to ours' GPU flash-attention kernel: CPU is at parity (ours 0.696 ≈ upstream ++ // 0.692) and `-fa off` on GPU recovers ours to 0.683, so the divergence is FA-kernel precision ++ // in the nextn draft's full-attn block, not this norm. RMSNorm is row-independent, so applying ++ // it before vs after the inp_out_ids reduction leaves t_embd / t_logits byte-identical — base ++ // decode + RULER unaffected. ++ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); ++ + cb(cur, "h_pre_norm", -1); + res->t_h_pre_norm = cur; + +@@ -222,9 +236,6 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + +- // Final norm +- cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); +- + cb(cur, "result_norm", -1); + res->t_embd = cur; + +@@ -539,9 +550,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + +- ggml_tensor * h_input = inp->embd; +- ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); +- cb(tok_embd, "mtp_tok_embd", il); ++ ggml_tensor * h_input = inp->embd; ++ ggml_tensor * inp_tokens = inp->tokens; + + res->add_input(std::move(inp)); + +@@ -549,6 +559,17 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + ggml_tensor * inp_out_ids = build_inp_out_ids(); + auto * inp_attn = build_attn_inp_kv(); + ++ // opencoti-hook: fused-nextn-mtp — one MTP step (embed → block → head → greedy argmax) as a ++ // reusable lambda the fused draft graph replays N× on-device (in-graph argmax→embed chain, ++ // one launch+readback). Takes the token index (embeds internally) so step k+1 can chain on ++ // step k's argmax. Single-step (n_mtp_steps==1) is byte-identical to the prior inline build ++ // (the argmax node is pruned when unreferenced). See docs/features/fused_nextn_mtp.md. ++ auto build_one_step = [&](ggml_tensor * h_input, ggml_tensor * tok, ++ ggml_tensor * inp_pos, auto * inp_attn, ++ ggml_tensor * inp_out_ids, bool fused) ++ -> std::tuple { ++ ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, tok); ++ cb(tok_embd, "mtp_tok_embd", il); + ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + +@@ -597,16 +618,27 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); +- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow); + + const float kq_scale = hparams.f_attention_scale == 0.0f + ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + +- cur = build_attn(inp_attn, +- nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ if (fused) { ++ // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the ++ // FROZEN prefix KV. The prefix K/V were already written (rope'd + normed) by the draft ++ // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the ++ // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used ++ // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. ++ // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); ++ } else { ++ Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++ ++ cur = build_attn(inp_attn, ++ nullptr, nullptr, nullptr, ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ } + cb(cur, "mtp_attn_pregate", il); + + cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate)); +@@ -634,7 +666,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + // Pre-norm hidden state: used by the AR draft loop to seed the next MTP step. + // (In the trunk graph this is `t_h_pre_norm`; the MTP head reuses the same slot.) + cb(cur, "h_pre_norm", -1); +- res->t_h_pre_norm = cur; ++ ggml_tensor * h_pre_norm = cur; + + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + +@@ -651,6 +683,70 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + +- res->t_logits = cur; +- ggml_build_forward_expand(gf, cur); ++ ggml_tensor * arg = ggml_argmax(ctx0, cur); ++ cb(arg, "mtp_argmax", -1); ++ ++ return { h_pre_norm, cur, arg }; ++ }; ++ ++ const int32_t n_steps = std::max(params.n_mtp_steps, 1); ++ ++ if (n_steps <= 1) { ++ auto [h_pre_norm, logits, arg] = build_one_step(h_input, inp_tokens, inp_pos, inp_attn, inp_out_ids, false); ++ (void) arg; // single-step: draft token comes from the backend sampler (t_sampled) or host sampler ++ res->t_h_pre_norm = h_pre_norm; ++ res->t_logits = logits; ++ ggml_build_forward_expand(gf, logits); ++ return; ++ } ++ ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant ++ // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 ++ // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] ++ // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, ++ // its seed hidden = step k's t_h_pre_norm — no host round-trip. Reuses llm_graph_input_mtp ++ // (width-agnostic inp_h_prev). See docs/features/fused_nextn_mtp.md. ++ const int64_t n_pos_e = hparams.n_pos_per_embd(); ++ auto inp_mtp = std::make_unique(); ++ inp_mtp->inp_last_token = inp_tokens; ++ inp_mtp->inp_h_prev = h_input; ++ inp_mtp->inp_pos_steps.reserve(n_steps); ++ std::vector pos_steps; ++ pos_steps.reserve(n_steps); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos_e); ++ ggml_set_input(p); ++ cb(p, "mtp_inp_pos_step", k); ++ pos_steps.push_back(p); ++ inp_mtp->inp_pos_steps.push_back(p); ++ } ++ res->add_input(std::move(inp_mtp)); ++ ++ ggml_tensor * tok_k = inp_tokens; ++ ggml_tensor * h_k = h_input; ++ std::vector step_args; ++ step_args.reserve(n_steps); ++ ggml_tensor * last_h_pre_norm = nullptr; ++ ggml_tensor * last_logits = nullptr; ++ for (int32_t k = 0; k < n_steps; ++k) { ++ auto [h_pre_norm_k, logits_k, arg_k] = build_one_step(h_k, tok_k, pos_steps[k], inp_attn, inp_out_ids, true); ++ step_args.push_back(arg_k); ++ last_h_pre_norm = h_pre_norm_k; ++ last_logits = logits_k; ++ tok_k = arg_k; // next token = this step's greedy argmax (I32[1]) ++ h_k = h_pre_norm_k; // next seed hidden = this step's pre-norm hidden ++ } ++ ++ ggml_tensor * all_args = step_args[0]; ++ for (int32_t k = 1; k < n_steps; ++k) { ++ all_args = ggml_concat(ctx0, all_args, step_args[k], 0); ++ } ++ cb(all_args, "mtp_fused_argmax", -1); ++ ++ res->t_argmax = all_args; // I32[n_steps] drafted tokens ++ res->t_h_pre_norm = last_h_pre_norm; // final hidden (seed for next cycle) ++ res->t_logits = last_logits; ++ ggml_build_forward_expand(gf, all_args); ++ ggml_build_forward_expand(gf, last_h_pre_norm); + } +diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp +index f3f8e73..0eb2871 100644 +--- a/llama.cpp/src/models/qwen35moe.cpp ++++ b/llama.cpp/src/models/qwen35moe.cpp +@@ -602,9 +602,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + +- ggml_tensor * h_input = inp->embd; +- ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); +- cb(tok_embd, "mtp_tok_embd", il); ++ ggml_tensor * h_input = inp->embd; ++ ggml_tensor * inp_tokens = inp->tokens; + + res->add_input(std::move(inp)); + +@@ -612,7 +611,17 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + ggml_tensor * inp_out_ids = build_inp_out_ids(); + auto * inp_attn = build_attn_inp_kv(); + +- ++ // opencoti-hook: fused-nextn-mtp — one MTP step (embed → block → head → greedy argmax) as a ++ // reusable lambda the fused draft graph replays N× on-device (in-graph argmax→embed chain, ++ // one launch+readback). Takes the token index (embeds internally) so step k+1 can chain on ++ // step k's argmax. Single-step (n_mtp_steps==1) is byte-identical to the prior inline build ++ // (the argmax node is pruned when unreferenced). See docs/features/fused_nextn_mtp.md. ++ auto build_one_step = [&](ggml_tensor * h_input, ggml_tensor * tok, ++ ggml_tensor * inp_pos, auto * inp_attn, ++ ggml_tensor * inp_out_ids, bool fused) ++ -> std::tuple { ++ ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, tok); ++ cb(tok_embd, "mtp_tok_embd", il); + ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + +@@ -661,16 +670,27 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); +- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, +- ext_factor, attn_factor, beta_fast, beta_slow); + + const float kq_scale = hparams.f_attention_scale == 0.0f + ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + +- cur = build_attn(inp_attn, +- nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ if (fused) { ++ // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the ++ // FROZEN prefix KV. The prefix K/V were already written (rope'd + normed) by the draft ++ // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the ++ // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used ++ // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. ++ // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); ++ } else { ++ Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, ++ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++ ++ cur = build_attn(inp_attn, ++ nullptr, nullptr, nullptr, ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ } + cb(cur, "mtp_attn_pregate", il); + + cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate)); +@@ -730,7 +750,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + + // Pre-norm hidden state: used by the AR draft loop to seed the next MTP step. + cb(cur, "h_pre_norm", -1); +- res->t_h_pre_norm = cur; ++ ggml_tensor * h_pre_norm = cur; + + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + +@@ -747,6 +767,70 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + +- res->t_logits = cur; +- ggml_build_forward_expand(gf, cur); ++ ggml_tensor * arg = ggml_argmax(ctx0, cur); ++ cb(arg, "mtp_argmax", -1); ++ ++ return { h_pre_norm, cur, arg }; ++ }; ++ ++ const int32_t n_steps = std::max(params.n_mtp_steps, 1); ++ ++ if (n_steps <= 1) { ++ auto [h_pre_norm, logits, arg] = build_one_step(h_input, inp_tokens, inp_pos, inp_attn, inp_out_ids, false); ++ (void) arg; // single-step: host samples via common_sampler_sample (argmax node pruned) ++ res->t_h_pre_norm = h_pre_norm; ++ res->t_logits = logits; ++ ggml_build_forward_expand(gf, logits); ++ return; ++ } ++ ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant ++ // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 ++ // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] ++ // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, ++ // its seed hidden = step k's t_h_pre_norm — no host round-trip. Reuses llm_graph_input_mtp ++ // (width-agnostic inp_h_prev). See docs/features/fused_nextn_mtp.md. ++ const int64_t n_pos_e = hparams.n_pos_per_embd(); ++ auto inp_mtp = std::make_unique(); ++ inp_mtp->inp_last_token = inp_tokens; ++ inp_mtp->inp_h_prev = h_input; ++ inp_mtp->inp_pos_steps.reserve(n_steps); ++ std::vector pos_steps; ++ pos_steps.reserve(n_steps); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos_e); ++ ggml_set_input(p); ++ cb(p, "mtp_inp_pos_step", k); ++ pos_steps.push_back(p); ++ inp_mtp->inp_pos_steps.push_back(p); ++ } ++ res->add_input(std::move(inp_mtp)); ++ ++ ggml_tensor * tok_k = inp_tokens; ++ ggml_tensor * h_k = h_input; ++ std::vector step_args; ++ step_args.reserve(n_steps); ++ ggml_tensor * last_h_pre_norm = nullptr; ++ ggml_tensor * last_logits = nullptr; ++ for (int32_t k = 0; k < n_steps; ++k) { ++ auto [h_pre_norm_k, logits_k, arg_k] = build_one_step(h_k, tok_k, pos_steps[k], inp_attn, inp_out_ids, true); ++ step_args.push_back(arg_k); ++ last_h_pre_norm = h_pre_norm_k; ++ last_logits = logits_k; ++ tok_k = arg_k; // next token = this step's greedy argmax (I32[1]) ++ h_k = h_pre_norm_k; // next seed hidden = this step's pre-norm hidden ++ } ++ ++ ggml_tensor * all_args = step_args[0]; ++ for (int32_t k = 1; k < n_steps; ++k) { ++ all_args = ggml_concat(ctx0, all_args, step_args[k], 0); ++ } ++ cb(all_args, "mtp_fused_argmax", -1); ++ ++ res->t_argmax = all_args; // I32[n_steps] drafted tokens ++ res->t_h_pre_norm = last_h_pre_norm; // final hidden (seed for next cycle) ++ res->t_logits = last_logits; ++ ggml_build_forward_expand(gf, all_args); ++ ggml_build_forward_expand(gf, last_h_pre_norm); + } +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 7bad046..34b50b1 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -965,10 +965,49 @@ private: + return false; + } + +- // No separate draft model/context: the assistant decodes on ctx_tgt via llama_decode_mtp. +- params_base.speculative.draft.ctx_tgt = ctx_tgt; +- params_base.speculative.draft.ctx_dft = nullptr; +- SRV_INF("%s", "MTP assistant loaded into target\n"); ++ // opencoti bug-858 dual-context MTP (A5): optionally create a REAL draft context FROM the ++ // nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 ++ // dual-context). The assistant is KV-less: its create_memory (A3) shares the target's KV ++ // cells + aliases the last full/SWA layer, and its graph (A6) reads the target's tok_embd ++ // + shared K/V via ctx_other; the shared-KV draft_mtp driver (is_mem_shared) then runs it. ++ // Gated by OPENCOTI_MTP_DUAL_CTX so the default keeps the proven single-context in-target ++ // path (ctx_dft == nullptr, llama_decode_mtp) byte-identical. ++ const char * mtp_dual_env = getenv("OPENCOTI_MTP_DUAL_CTX"); ++ const bool mtp_dual_ctx = mtp_dual_env && mtp_dual_env[0] && mtp_dual_env[0] != '0'; ++ if (mtp_dual_ctx) { ++ // llama_init_from_model wants a mutable model*; the nested assistant is owned mutably ++ // by the target (unique_ptr), so the const_cast is safe. ++ llama_model * assistant = const_cast(llama_model_get_mtp_assistant(model_tgt)); ++ if (assistant == nullptr) { ++ SRV_ERR("%s", "MTP dual-context requested but the assistant is not loaded into the target\n"); ++ return false; ++ } ++ ++ auto cparams_mtp = common_context_params_to_llama(params_dft); ++ cparams_mtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; ++ cparams_mtp.type_k = params_spec.cache_type_k; ++ cparams_mtp.type_v = params_spec.cache_type_v; ++ cparams_mtp.n_rs_seq = 0; ++ cparams_mtp.ctx_other = ctx_tgt; ++ ++ ctx_dft.reset(llama_init_from_model(assistant, cparams_mtp)); ++ if (ctx_dft == nullptr) { ++ SRV_ERR("%s", "failed to create dual-context MTP assistant draft context\n"); ++ return false; ++ } ++ ++ // NOTE: leave ctx_dft_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO (the default) — the ++ // assistant's KV is SHARED with the target, so ctx_dft must never be independently ++ // seq_rm'd / checkpointed (that would corrupt the target's cells). ++ params_base.speculative.draft.ctx_tgt = ctx_tgt; ++ params_base.speculative.draft.ctx_dft = ctx_dft.get(); ++ SRV_INF("%s", "MTP assistant dual-context draft created (ctx_other=target, shared KV)\n"); ++ } else { ++ // No separate draft context: the assistant decodes on ctx_tgt via llama_decode_mtp. ++ params_base.speculative.draft.ctx_tgt = ctx_tgt; ++ params_base.speculative.draft.ctx_dft = nullptr; ++ SRV_INF("%s", "MTP assistant loaded into target (single-context)\n"); ++ } + } else if (params_base.speculative.has_dft()) { + // TODO speculative: move to common/speculative.cpp? + const auto & params_spec = params_base.speculative.draft; +@@ -2641,9 +2710,7 @@ private: + } + + // generate the actual drafts (if any) +- { +- common_speculative_draft(spec.get()); +- } ++ common_speculative_draft(spec.get()); + + // make checkpoints if needed + for (auto * slot_ptr : drafting) { +@@ -2674,13 +2741,8 @@ private: + (ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_dft.get())); + + if (use_ckpt_tgt) { +- //const int64_t t_start = ggml_time_us(); +- + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + +- //const int64_t t_total = ggml_time_us() - t_start; +- //printf("checkpoint total: %f ms\n", t_total / 1000.0); +- + SLT_DBG(slot, "created speculative checkpoint (pos_min = %d, pos_max = %d, n_tokens = %d, size = %.3f MiB, draft = %.3f MiB)\n", + ckpt.pos_min, ckpt.pos_max, slot.prompt.n_tokens(), + (float) ckpt.size() / 1024 / 1024, diff --git a/patches/0094-tinyblas-small-gemm.patch b/patches/0094-tinyblas-small-gemm.patch new file mode 100644 index 0000000000000000000000000000000000000000..ce870d287bf355cc137f2db0353334619a532223 --- /dev/null +++ b/patches/0094-tinyblas-small-gemm.patch @@ -0,0 +1,31 @@ +From: opencoti +Subject: [PATCH 0094] tinyblas small-GEMM second config (bug-2104, #614) + +opencoti-hook: tinyblas-small-gemm — skinny f32 GEMMs (Gemma MoE router +ffn_gate_inp [2816x128], qwen35 DeltaNet chunk GEMMs) hit a single 128x64-tile +config = 8 blocks on a 142-SM RTX 6000 (5% occupancy, 274us). Add a second +<32,32,64,...> config selected when the big config yields <24 blocks (90.7us; +prefill +25-35% on MoE models). + +diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu +index 758f68a..76ba099 100644 +--- a/llamafile/tinyblas.cu ++++ b/llamafile/tinyblas.cu +@@ -539,6 +539,17 @@ tinyblasStatus_t tinyblasGE_launch(tinyblasHandle_t handle, tinyblasOperation_t + int ldc) { + if (can_use_matvec(aT, bT, m, n, k, alpha, beta)) + return matvec_launch(handle, m, k, A, lda, B, C); ++ // opencoti-hook: tinyblas-small-gemm (#614/bug-2104) — the single 128x64-tile config ++ // launches only ceil(m/64)*ceil(n/128) blocks after the row-major swap below; a skinny ++ // GEMM like the Gemma MoE router ([2816x128] x [2816x512] per layer per ubatch) gets 8 ++ // blocks on a 142-SM GPU (~6% occupancy, ~274 us vs cuBLAS ~8 us) and was 35% of A4B ++ // prefill GPU time. When the big config cannot fill the GPU, use a 32x32-tile config ++ // (same deterministic FFMA warp2d kernel, 4x1 thread tiles, 2 warps) so the block ++ // count grows 8x. Threshold 24 keeps every launch that already saturates on the big ++ // tiles untouched. ++ if ((long long)CEIL_DIV(m, 64) * CEIL_DIV(n, 128) < 24) ++ return tinyblasGE_launcher<32, 32, 64, 8, 32, 16, 1, 4, 4, 64>( ++ handle, bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc); + constexpr int TT = 256; + constexpr int BM = 128; + constexpr int BN = 64; diff --git a/patches/0095-prefill-checkpoint-align.patch b/patches/0095-prefill-checkpoint-align.patch new file mode 100644 index 0000000000000000000000000000000000000000..ead2b8384bf9f836ff47a9656f6cd181934dda6b --- /dev/null +++ b/patches/0095-prefill-checkpoint-align.patch @@ -0,0 +1,145 @@ +From: opencoti +Subject: [PATCH 0095] prefill checkpoint alignment: min-step 8192 + state-buffer recycle pool (bug-2105/2107, #614) + +- checkpoint_min_step 256 -> 8192 (align with b9859; stale port default caused + several SWA/recurrent state checkpoints per request instead of <=1). +- opencoti-hook: ckpt-buf-noinit — common_state_buf type + server-side recycle + pool (ckpt_buf_pool, cap 4) so steady-state requests reuse warm pages: no + GB-class value-init memset and no page-fault storm inside the state D2H copy. + (Measured dead end: a non-zeroing allocator alone regressed all models 8-13%.) + +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index b688492..3bb6aa2 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -670,7 +673,7 @@ struct common_params { + bool cache_prompt = true; // whether to enable prompt caching + bool cache_idle_slots = true; // save and clear idle slots upon starting a new task + int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot +- int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints ++ int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints + int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. + + std::string hostname = "127.0.0.1"; +@@ -1124,14 +1127,21 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); + // prompt utils + // + ++// opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — named buffer type for state snapshots so ++// the server can recycle them across requests (warm pages: no memset, no page-fault storm). ++// Measured dead end kept for the record: a non-zeroing allocator here made the COLD path ++// slower (page faults land inside the state D2H copy loop instead of a fast sequential ++// memset prefault) — plain vector cold + recycled warm is the optimum. ++using common_state_buf = std::vector; ++ + struct common_prompt_checkpoint { + int64_t n_tokens; + + llama_pos pos_min; + llama_pos pos_max; + +- std::vector data_tgt; +- std::vector data_dft; ++ common_state_buf data_tgt; ++ common_state_buf data_dft; + + size_t size() const; + +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 7bad046..34b50b1 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -2119,20 +2158,50 @@ private: + return true; + } + ++ // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — recycle retired checkpoint state ++ // buffers so the pages stay mapped and warm; a fresh GB-class allocation per request ++ // pays a page-fault storm inside the state D2H copy (worse than the memset it replaced) ++ std::vector ckpt_buf_pool; ++ ++ void ckpt_buf_recycle(common_prompt_checkpoint & ckpt) { ++ if (ckpt_buf_pool.size() < 4 && ckpt.data_tgt.capacity() > 0) { ++ ckpt_buf_pool.push_back(std::move(ckpt.data_tgt)); ++ } ++ if (ckpt_buf_pool.size() < 4 && ckpt.data_dft.capacity() > 0) { ++ ckpt_buf_pool.push_back(std::move(ckpt.data_dft)); ++ } ++ } ++ ++ common_state_buf ckpt_buf_acquire() { ++ if (ckpt_buf_pool.empty()) { ++ return {}; ++ } ++ common_state_buf buf = std::move(ckpt_buf_pool.back()); ++ ckpt_buf_pool.pop_back(); ++ return buf; ++ } ++ + // n_tokens_cur: the number of tokens added to the batch for the current slot + void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { + while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { + // make room for the new checkpoint, if needed +- const auto & cur = slot.prompt.checkpoints.front(); ++ auto & cur = slot.prompt.checkpoints.front(); + + SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", + cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); + ++ ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) + slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); + } + + auto & cur = slot.prompt.checkpoints.emplace_back(); + ++ // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — seed with warm recycled buffers ++ cur.data_tgt = ckpt_buf_acquire(); ++ if (ctx_dft) { ++ cur.data_dft = ckpt_buf_acquire(); ++ } ++ + cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); + + cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); +@@ -2980,9 +3042,10 @@ private: + { + // erase any checkpoints with pos_max > pos_next + for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { +- const auto & cur = *it; ++ auto & cur = *it; + if (cur.pos_max > pos_next) { + SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); ++ ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) + it = slot.prompt.checkpoints.erase(it); + } else { + ++it; +diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp +index 803e31f..70ca8dc 100644 +--- a/llama.cpp/tools/server/server-task.cpp ++++ b/llama.cpp/tools/server/server-task.cpp +@@ -2025,8 +2025,8 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t + } + } + +- std::vector state_data_tgt; +- std::vector state_data_dft; ++ common_state_buf state_data_tgt; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) ++ common_state_buf state_data_dft; + + // check if we can allocate enough memory for the new state + try { +diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h +index 933bd99..380dc27 100644 +--- a/llama.cpp/tools/server/server-task.h ++++ b/llama.cpp/tools/server/server-task.h +@@ -582,8 +582,8 @@ struct server_task_result_apply_lora : server_task_result { + }; + + struct server_prompt_data { +- std::vector main; +- std::vector drft; ++ common_state_buf main; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) ++ common_state_buf drft; + + size_t size() const { + return main.size() + drft.size(); diff --git a/patches/0096-remove-single-ctx-gemma-mtp.patch b/patches/0096-remove-single-ctx-gemma-mtp.patch new file mode 100644 index 0000000000000000000000000000000000000000..cab18cf5db4aac904213b5b9d19298c268e986aa --- /dev/null +++ b/patches/0096-remove-single-ctx-gemma-mtp.patch @@ -0,0 +1,2308 @@ +From: opencoti +Subject: [PATCH 0096] remove single-context Gemma MTP facade; dual-ctx is the only assistant path (#611) + +END of the bug-858 port: delete the retired single-context in-target assistant +engine (~1600 lines) now that the dual-context port (0093) proved accept parity: +- llama-context: decode_mtp facade + decode_mtp_sync/fused, async worker + (decode_mtp_async/wait/run, mtp_worker thread, drain guard), process_ubatch_mtp, + graph_params_mtp, ensure_sched_mtp, sched_mtp/gf_res_prev_mtp, graph_compute_mtp. +- public API llama_decode_mtp removed (llama_decode_mtp_fused_nextn stays). +- llama-graph: build_attn_mtp (iswa cross-read); kv-cache-iswa::init_mtp. +- models: llm_build_gemma4_mtp + one-step builder + gemma4.cpp MTP dispatch; + gemma4-assistant keeps ONLY the dual-ctx graph (ctx_other required). +- speculative: common_speculative_impl_draft_assistant + NDJSON tracer removed; + --spec-type draft-assistant now requires ctx_dft (server always creates it). +- server: OPENCOTI_MTP_DUAL_CTX env gate removed — dual-ctx unconditional; + the --parallel>1 -> --kv-unified boot guard is preserved (shared-KV aliasing + has the same per-stream-split constraint the facade had). +NextN side untouched (mtp_fused_steps, sched_nextn, mtp_slot_info, unified twin +builders); mtp_assistant model loading kept. Gate on bs2: A4B n2 305.4 tps / +0.849 accept (== 0093 reference, boot line proves dual-ctx default), q35 n3 +248.8 / 0.662. bug-2110 (par2 accept 0.635 on the tps harness) was RESOLVED as +a harness artifact: on the real 9-prompt mtp-bench the dual-ctx assistant holds +accept at --parallel 2 --kv-unified (ours 0.785 par1 / 0.783 par2; upstream +b9859 0.793 / 0.792). + +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index 3bb6aa2..887dfb6 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -305,7 +305,7 @@ struct common_params_speculative_draft { + + // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): OPTIONAL explicit draft-depth ceiling. + // 0 (default) = draft depth follows --spec-draft-n-max (n_max); the MTP head re-runs +- // autoregressively per step (decode_mtp_sync/fused), so depth is NOT limited to a fixed ++ // autoregressively per step (draft_mtp / decode_mtp_fused_nextn), so depth is NOT limited to a fixed + // block size. bug-858: the old default 3 hard-capped ours at 2 drafts/round and blocked + // n-max scaling vs upstream b9859. When set >1, caps draft depth at draft_block_size-1. + int32_t draft_block_size = 0; +diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp +index ee7962f..010bf65 100644 +--- a/llama.cpp/common/speculative.cpp ++++ b/llama.cpp/common/speculative.cpp +@@ -1409,315 +1409,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { + } + }; + +-// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. +-// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, +-// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV +-// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens +-// of a block in ONE facade call (no per-step sampler loop). +-// +-// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), +-// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never +-// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() +-// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) +-// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). +- +-// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. +-// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. +-namespace { +-struct mtp_acc_tracer { +- bool enabled = false; +- FILE * fp = nullptr; +- std::mutex mu; +- +- mtp_acc_tracer() { +- const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); +- if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { +- return; +- } +- fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); +- enabled = (fp != nullptr); +- } +- +- ~mtp_acc_tracer() { +- if (fp && fp != stderr) { +- std::fclose(fp); +- } +- } +- +- void writeln(const std::string & line) { +- if (!enabled) { +- return; +- } +- std::lock_guard lk(mu); +- std::fputs(line.c_str(), fp); +- std::fputc('\n', fp); +- std::fflush(fp); +- } +-}; +- +-mtp_acc_tracer & mtp_tracer() { +- static mtp_acc_tracer t; +- return t; +-} +-} // namespace +- +-struct common_speculative_impl_draft_assistant : public common_speculative_impl { +- common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) +- +- llama_context * ctx_tgt = nullptr; +- +- int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes +- int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row +- +- // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows +- // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds +- // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h +- // scheme so multi-seq (PolyKV) works without per-seq server plumbing. +- std::vector> pending_h; // [n_seq][n_embd_backbone] +- std::vector> verify_h; // [n_seq][n_rows * n_embd_backbone] +- std::vector verify_h_rows; +- std::vector i_batch_beg; +- std::vector i_batch_end; +- +- // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head +- // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain +- // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. +- int skip_streak_threshold = 0; +- std::vector zero_accept_streak; +- std::vector skip_last_draft; +- std::vector prev_n_acc_at_draft; +- +- int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change +- +- static float compute_h_l2(const float * h, int32_t n) { +- if (!h || n <= 0) { +- return 0.0f; +- } +- double s = 0.0; +- for (int32_t i = 0; i < n; ++i) { +- s += (double) h[i] * (double) h[i]; +- } +- return (float) std::sqrt(s); +- } +- +- common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) +- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) +- , params(params.draft) +- { +- ctx_tgt = this->params.ctx_tgt; +- GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); +- +- const llama_model * model_tgt = llama_get_model(ctx_tgt); +- n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); +- n_embd_out = llama_model_n_embd_out(model_tgt); +- +- LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); +- LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, +- this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); +- +- // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. +- llama_set_embeddings(ctx_tgt, true); +- +- if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { +- const int v = std::atoi(e); +- if (v >= 1 && v <= 32) { +- skip_streak_threshold = v; +- } +- } +- +- pending_h.assign(n_seq, std::vector((size_t) std::max(1, n_embd_backbone), 0.0f)); +- verify_h.assign(n_seq, {}); +- verify_h_rows.assign(n_seq, 0); +- i_batch_beg.assign(n_seq, -1); +- i_batch_end.assign(n_seq, -1); +- zero_accept_streak.assign(n_seq, 0); +- skip_last_draft.assign(n_seq, 0); +- prev_n_acc_at_draft.assign(n_seq, 0); +- } +- +- ~common_speculative_impl_draft_assistant() override = default; +- +- void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { +- llama_set_embeddings(ctx_tgt, true); +- if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { +- skip_last_draft[seq_id] = 0; +- zero_accept_streak[seq_id] = 0; +- } +- } +- +- // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() +- // can select the last-accepted token's hidden state. The spec framework calls process() after the +- // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which +- // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. +- // NOTE (bug-858): PRE-norm harvest was TESTED and REFUTED — it collapses accept to ~0.05 at ALL +- // positions (incl short ctx). This drafter expects the post-norm hidden; do NOT swap to pre_norm. +- bool process(const llama_batch & batch_in) override { +- if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { +- return true; +- } +- if (n_embd_backbone <= 0) { +- return true; +- } +- +- const int32_t n_tokens = batch_in.n_tokens; +- std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); +- std::fill(i_batch_end.begin(), i_batch_end.end(), -1); +- +- for (int k = 0; k < n_tokens; ++k) { +- const llama_seq_id s = batch_in.seq_id[k][0]; +- if (s < 0 || s >= (llama_seq_id) n_seq) { +- continue; +- } +- i_batch_end[s] = k; +- if (i_batch_beg[s] < 0) { +- i_batch_beg[s] = k; +- } +- } +- +- const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); +- const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); +- +- for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { +- if (i_batch_beg[s] < 0) { +- continue; +- } +- const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; +- verify_h_rows[s] = n_rows; +- verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); +- +- for (int32_t i = 0; i < n_rows; ++i) { +- const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); +- if (h) { +- std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, +- (size_t) n_copy * sizeof(float)); +- } +- } +- // default pending_h to the last row (correct when all drafts accepted / on prefill) +- std::memcpy(pending_h[s].data(), +- verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); +- } +- +- return true; +- } +- +- void draft(common_speculative_draft_params_vec & dparams) override { +- if (n_embd_backbone <= 0) { +- return; +- } +- +- // bug-858: draft depth follows --spec-draft-n-max (n_max) and the room left in context +- // (dp.n_max = n_draft_max), NOT draft_block_size. The MTP head re-runs autoregressively per +- // step (decode_mtp_sync/fused build a fresh single-step graph each step: argmax -> next +- // last_token, h_post -> next h_prev), so depth is bounded by n_max, not a fixed block. The +- // legacy draft_block_size-1 ceiling (default 3 -> 2) hard-capped ours at 2 drafts/round and +- // blocked n-max scaling vs upstream b9859 (which drafts to n_max autoregressively) -> REMOVED. +- +- for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { +- auto & dp = dparams[seq_id]; +- if (!dp.drafting) { +- continue; +- } +- +- // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, +- // the previous batch produced 0 accepted drafts. +- if (skip_last_draft[seq_id]) { +- skip_last_draft[seq_id] = 0; +- } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { +- zero_accept_streak[seq_id]++; +- } else { +- zero_accept_streak[seq_id] = 0; +- } +- if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { +- zero_accept_streak[seq_id] = 0; +- skip_last_draft[seq_id] = 1; +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- continue; // empty result -> server falls back to single-token verify +- } +- +- int32_t n_steps = params.n_max > 0 ? params.n_max : 1; +- if (dp.n_max > 0) { +- n_steps = std::min(n_steps, dp.n_max); +- } +- if (n_steps <= 0) { +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- continue; +- } +- +- float * h_prev = pending_h[seq_id].data(); +- +- llama_memory_t mem = llama_get_memory(ctx_tgt); +- llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; +- if (attn_pos < 0) { +- attn_pos = 0; +- } +- +- std::vector out((size_t) n_steps, 0); +- const int32_t rc = llama_decode_mtp( +- ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, +- /*out_drafts =*/ out.data(), +- /*out_logits =*/ nullptr, +- /*out_h_prev_last=*/ nullptr); +- +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- +- if (rc != 0) { +- LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); +- continue; +- } +- +- auto & result = *dp.result; +- for (int32_t i = 0; i < n_steps; ++i) { +- result.push_back(out[i]); +- } +- +- if (mtp_tracer().enabled) { +- std::ostringstream oss; +- oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter +- << ",\"seq_id\":" << (int) seq_id +- << ",\"id_last\":" << (int) dp.id_last +- << ",\"attn_pos\":" << (int) attn_pos +- << ",\"n_steps\":" << n_steps +- << ",\"h_l2\":" << std::fixed << std::setprecision(4) +- << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; +- for (int32_t i = 0; i < n_steps; ++i) { +- if (i) oss << ','; +- oss << (int) out[i]; +- } +- oss << "]}"; +- mtp_tracer().writeln(oss.str()); +- } +- } +- } +- +- void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { +- if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { +- return; +- } +- const int32_t n_rows = verify_h_rows[seq_id]; +- if (n_rows <= 0 || n_embd_backbone <= 0) { +- return; +- } +- // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the +- // last-accepted token's hidden state (clamped to the harvested rows). +- const int32_t i_h = std::min(n_accepted, n_rows - 1); +- std::memcpy(pending_h[seq_id].data(), +- verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, +- (size_t) n_embd_backbone * sizeof(float)); +- +- if (mtp_tracer().enabled) { +- std::ostringstream oss; +- oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter +- << ",\"seq_id\":" << (int) seq_id +- << ",\"n_accepted\":" << (int) n_accepted << "}"; +- mtp_tracer().writeln(oss.str()); +- ++trace_iter; +- } +- } +- +- bool need_embd() const override { +- return true; // MTP reads POST-norm output embeddings (h_prev) +- } +-}; + + struct common_speculative { + common_speculative_draft_params_vec dparams; +@@ -1934,15 +1625,15 @@ common_speculative * common_speculative_init(common_params_speculative & params, + break; + } + case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter +- // opencoti bug-858 dual-context MTP: when the server created a real ctx_dft FROM the +- // assistant model (ctx_other=ctx_tgt; --spec-type draft-assistant + OPENCOTI_MTP_DUAL_CTX), +- // run the shared-KV draft_mtp driver (upstream b9859 is_mem_shared). Otherwise fall +- // back to the proven single-context in-target assistant (ctx_dft == nullptr). +- if (config.params.draft.ctx_dft != nullptr) { +- impls.push_back(std::make_unique(config.params, n_seq)); +- } else { +- impls.push_back(std::make_unique(config.params, n_seq)); ++ // opencoti bug-858 dual-context MTP (#611): the assistant ALWAYS runs as a real ++ // ctx_dft created FROM the assistant model (ctx_other=ctx_tgt) through the shared-KV ++ // draft_mtp driver (upstream b9859 is_mem_shared). The single-context in-target ++ // facade was removed in #611 after the dual-context port proved accept parity. ++ if (config.params.draft.ctx_dft == nullptr) { ++ LOG_ERR("%s: draft-assistant requires the dual-context draft (server creates ctx_dft with ctx_other=target)\n", __func__); ++ return nullptr; + } ++ impls.push_back(std::make_unique(config.params, n_seq)); + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { +@@ -2010,8 +1701,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, + // opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / + // set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative + // framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ +-// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on +-// common_speculative_impl_draft_assistant above). ++// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete. + + void common_speculative_free(common_speculative * spec) { + if (spec == nullptr) { +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index e6517be..2f4da86 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -1041,22 +1041,6 @@ extern "C" { + struct llama_context * ctx, + struct llama_batch batch); + +- // opencoti F5 M6-S4 mtp +- // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, +- // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). +- // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) +- // receive per-step logits and the final backbone hidden. Returns 0 on success. +- LLAMA_API int32_t llama_decode_mtp( +- struct llama_context * ctx, +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- + // opencoti fused-NextN (#590): fuse N per-step NextN draft decodes into one graph on a + // DECODER_MTP (Qwen NextN) draft context. GREEDY-ONLY (no p_min early-stop). Returns 0 on + // success. last_token must already be resident in ctx's cache at attn_pos (seed decode first). +diff --git a/llama.cpp/src/llama-arch.h b/llama.cpp/src/llama-arch.h +index 9aef864..85b1e98 100644 +--- a/llama.cpp/src/llama-arch.h ++++ b/llama.cpp/src/llama-arch.h +@@ -570,8 +570,8 @@ enum llm_tensor { + + // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 + // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile +- // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our +- // single-context build_attn_mtp cross-read — NOT upstream's ctx_other. See plan. ++ // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine is the ++ // bug-858 dual-context port (cparams.ctx_other, upstream-shaped). See plan. + LLM_TENSOR_NEXTN_PROJ_PRE, + LLM_TENSOR_NEXTN_PROJ_POST, + LLM_TENSOR_MASKED_EMBD_CENTROIDS, +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 7d27212..1579acc 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -9,7 +9,7 @@ + #include "llama-memory.h" + #include "llama-mmap.h" + #include "llama-model.h" +-#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp ++#include "llama-kv-cache-iswa.h" + #include "llama-ext.h" + #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" +@@ -527,17 +527,6 @@ llama_context::llama_context( + } + + llama_context::~llama_context() { +- // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down +- // sched_mtp and the backends it touches. +- if (mtp_worker.joinable()) { +- { +- std::lock_guard lk(mtp_mu); +- mtp_worker_stop.store(true, std::memory_order_release); +- } +- mtp_cv_request.notify_all(); +- mtp_worker.join(); +- } +- + if (!model.hparams.no_alloc) { + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; +@@ -1421,208 +1410,6 @@ bool llama_context::set_adapter_cvec( + return res; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-bool llama_context::ensure_sched_mtp() { +- // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count +- // (mtp_fused_steps). A fused N-step graph has ~N× the nodes of the single-step graph and a +- // distinct topology, so when a larger block is requested we drop and rebuild the reserve. +- const int32_t need = std::max(mtp_fused_steps, 1); +- if (sched_mtp && mtp_reserved_steps >= need) { +- return true; +- } +- if (!model.mtp_assistant) { +- return false; +- } +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- +- const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); +- const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; +- +- gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); +- sched_mtp.reset(ggml_backend_sched_new( +- backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), +- max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); +- if (!sched_mtp) { +- LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. +- // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), +- // so a single reserve covers all subsequent decode_mtp calls. +- { +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- llama_memory_context_ptr mctx = memory->init_full(); +- if (!mctx) { +- LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(0); +- data->seq_idx[0] = 0; +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = 0; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- const uint32_t save_n_outputs = n_outputs; +- n_outputs = 1; +- +- auto * res = gf_res_prev_mtp.get(); +- const auto gparams = graph_params_mtp(res, ub, mctx.get()); +- res->reset(); +- ggml_backend_sched_reset(sched_mtp.get()); +- +- auto * gf = model.build_graph(gparams); +- n_outputs = save_n_outputs; +- +- if (!gf) { +- LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { +- LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- // Discard the reserve graph cache so the first real call rebuilds with proper inputs. +- res->reset(); +- ggml_backend_sched_reset(sched_mtp.get()); +- } +- +- // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in +- // decode_mtp_async (only when the opt-in async path is actually taken), NOT here — so the +- // default in-thread sync path creates no extra thread and is runtime-identical to P3. +- +- mtp_reserved_steps = need; // reserve now covers `need` fused steps +- return true; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-llm_graph_result * llama_context::process_ubatch_mtp( +- const llama_ubatch & ubatch, +- llama_memory_context_i * mctx, +- ggml_status & ret) { +- GGML_ASSERT(sched_mtp && gf_res_prev_mtp); +- +- auto * res = gf_res_prev_mtp.get(); +- auto * gf = res->get_gf(); +- +- // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it +- // explicitly here in case a concurrent target decode mutates it. The MTP graph +- // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. +- llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); +- gparams.n_outputs = 1; +- gparams.sched = sched_mtp.get(); +- +- if (!graph_reuse_disable && res->can_reuse(gparams)) { +- // sched_mtp does not use pipeline parallelism (created with pipeline=false), +- // so no synchronize is needed before set_inputs. +- } else { +- res->reset(); +- +- ggml_backend_sched_reset(sched_mtp.get()); +- ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); +- +- gf = model.build_graph(gparams); +- if (!gf) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); +- ret = GGML_STATUS_FAILED; +- return nullptr; +- } +- if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { +- LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); +- ret = GGML_STATUS_ALLOC_FAILED; +- return nullptr; +- } +- } +- +- res->set_inputs(&ubatch); +- +- const auto status = graph_compute_mtp(res->get_gf()); +- +- if (status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); +- ret = status; +- return nullptr; +- } +- +- ret = GGML_STATUS_SUCCESS; +- return res; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { +- // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s +- // threadpool/n_threads dance only for the CPU backend; the MTP head is small and +- // thread saturation matters less, but consistency with graph_compute() avoids +- // surprising backend reconfigurations between target and MTP runs. +- // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker +- // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. +- const int n_threads = cparams.n_threads; +- ggml_threadpool_t tp = threadpool; +- +- if (backend_cpu != nullptr) { +- auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); +- auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) +- ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); +- if (set_threadpool_fn) { +- set_threadpool_fn(backend_cpu, tp); +- } +- } +- for (const auto & set_n_threads_fn : set_n_threads_fns) { +- set_n_threads_fn.second(set_n_threads_fn.first, n_threads); +- } +- +- auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); +- if (status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", +- __func__, status); +- } +- return status; +-} +- + llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { + if (mctx && !mctx->apply()) { + LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); +@@ -2093,14 +1880,6 @@ int llama_context::decode(const llama_batch & batch_inp) { + + bool did_optimize = false; + +- // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its +- // KV below (memory_update's pending shifts/copies, then init_batch's slot apply — the S1 +- // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this +- // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in +- // the current sync-facade integration (nothing is in flight here) and a single pointer +- // check when no MTP assistant is loaded. +- mtp_drain_before_mutate(); +- + // handle any pending shifts/copies + memory_update(false); + +@@ -2684,368 +2463,11 @@ ggml_cgraph * llama_context::graph_reserve( + return gf; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-llm_graph_params llama_context::graph_params_mtp( +- llm_graph_result * res, +- const llama_ubatch & ubatch, +- const llama_memory_context_i * mctx) const { +- const llama_model * mtp = model.mtp_assistant.get(); +- GGML_ASSERT(mtp); +- +- return { +- /*.arch =*/ mtp->arch, +- /*.hparams =*/ mtp->hparams, +- /*.cparams =*/ cparams, +- /*.ubatch =*/ ubatch, +- /*.gtype =*/ LLM_GRAPH_TYPE_MTP, +- /*.sched =*/ sched.get(), +- /*.backend_cpu =*/ backend_cpu, +- /*.cvec =*/ cvec.get(), +- /*.loras =*/ loras.get(), +- /*.mctx =*/ mctx, +- /*.cross =*/ &cross, +- /*.samplers =*/ sampling.samplers, +- /*.n_outputs =*/ n_outputs, +- /*.cb =*/ graph_get_cb(), +- /*.res =*/ res, +- /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) +- }; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. +-int32_t llama_context::decode_mtp( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams +- // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the +- // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes async→wait. +- // With no target/draft overlap this is byte-identical to the P3 sync path — the worker runs +- // the same N-step loop off-thread while we block in decode_mtp_wait. +- if (out_logits) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_logits, out_h_prev_last); +- } +- // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the +- // fused single-graph draft (decode_mtp_fused) — ONE launch+sync+D2H for the whole block, the +- // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused +- // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse → sequential. +- static const bool fused_enabled = [] { +- const char * e = getenv("OPENCOTI_MTP_FUSED"); +- return !(e && e[0] == '0'); +- }(); +- if (fused_enabled && n_steps > 1) { +- return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_h_prev_last); +- } +- // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). +- // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker +- // gives NO throughput benefit — real depth-2 overlap is deferred (limited by the h-prev +- // dependency; even the AtomicBot fork ships depth-1) — AND the worker-thread CUDA execution +- // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off +- // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. +- static const bool async_enabled = [] { +- const char * e = getenv("OPENCOTI_MTP_ASYNC"); +- return e && e[0] == '1'; +- }(); +- if (!async_enabled) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_logits, out_h_prev_last); +- } +- const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); +- if (rc != 0) { +- return rc; +- } +- return decode_mtp_wait(out_drafts, out_h_prev_last); +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-int32_t llama_context::decode_mtp_sync( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- +- // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, +- // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) +- mtp_fused_steps = 1; +- +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- +- const int32_t n_vocab = model.vocab.n_tokens(); +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0) { +- LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); +- return -4; +- } +- +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(seq_id); +- data->seq_idx[(size_t) seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post +- // back). It MUST be flagged output=1 so build_inp_out_ids selects its row — and so the live +- // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here +- // produced a divergent graph whose output-selection tensor was left with a null backend +- // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. +- data->output[0] = 1; +- +- if (n_steps <= 0) { +- return 0; +- } +- +- // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token +- // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. +- for (int32_t k = 0; k < n_steps; ++k) { +- data->token[0] = last_token; +- // bug-858: the gemma4-assistant (is_mem_shared) drafts ALL steps from the SAME query +- // position — the head is trained to predict multiple future tokens from one position, with +- // the recurrence carried by the h_prev hidden chain (NOT the RoPE angle). Incrementing the +- // position per step (attn_pos+1+k) hands the head a mistrained RoPE angle from step 1 on, +- // degrading step-1+ drafts (aggregate accept ~0.69 vs upstream 0.79). Match b9859 draft_mtp +- // is_mem_shared: common_batch_add(batch, id, dp.n_past, ...) — constant dp.n_past = attn_pos+1. +- data->pos[0] = attn_pos + 1; +- std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); +- +- int32_t best_i32 = 0; +- ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); +- out_drafts[k] = (llama_token) best_i32; +- +- if (out_logits) { +- ggml_tensor * t_logits = res->get_logits(); +- GGML_ASSERT(t_logits); +- ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, +- 0, (size_t) n_vocab * sizeof(float)); +- } +- +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); +- +- last_token = (llama_token) best_i32; +- } +- +- if (out_h_prev_last) { +- std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); +- } +- +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head +-// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls +-// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index — already +-// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. +-// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, +-// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). +-// +-// One mask shared across steps (built from step 0's position). For FULL-attn target layers and +-// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa — the short-context / +-// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are +-// byte-identical to the sequential path. For SWA layers at long context the true per-step window +-// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is +-// output-safe — the target verify pass rejects any mismatched draft, so the emitted stream is +-// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix +-// is a per-step mask slice — deferred until measured. See buglog bug-420.) +-int32_t llama_context::decode_mtp_fused( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_h_prev_last) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- if (n_steps <= 0) { +- return 0; +- } +- // A single step has nothing to fuse — defer to the proven sequential path. +- if (n_steps == 1) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, /*out_logits=*/nullptr, out_h_prev_last); +- } +- +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0) { +- LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); +- return -4; +- } +- +- // Build/reserve a fused n_steps graph. +- mtp_fused_steps = n_steps; +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- +- // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step +- // query positions consumed by the wrapper's inp_pos_steps (set_input). +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(n_steps); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(seq_id); +- data->seq_idx[(size_t) seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = (uint32_t) n_steps; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; +- +- data->token[0] = last_token; +- std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); +- // bug-858: constant query position for ALL draft steps (see decode_mtp_sync note). The +- // gemma4-assistant (is_mem_shared) is trained to predict every future draft token from the +- // SAME position (dp.n_past = attn_pos+1); recurrence lives in the h_prev chain, not the RoPE +- // angle. The old attn_pos+1+k mistrained step-1+ RoPE -> accept 0.69 vs upstream 0.79. +- for (int32_t k = 0; k < n_steps; ++k) { +- data->pos[k] = attn_pos + 1; +- } +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); +- GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); +- +- std::vector drafts((size_t) n_steps); +- ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); +- for (int32_t k = 0; k < n_steps; ++k) { +- out_drafts[k] = (llama_token) drafts[(size_t) k]; +- } +- +- if (out_h_prev_last) { +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); +- } +- +- return 0; +-} +- + // opencoti fused-NextN (#590 / bug-858): fuse the N per-step NextN draft decodes into ONE graph on + // the draft context (ctx_dft, type LLAMA_CONTEXT_TYPE_MTP → LLM_GRAPH_TYPE_DECODER_MTP). This is the + // throughput fix: the un-fused path pays a host sync per draft step; here one graph emits N greedy +-// drafts with a single sync. Twin of decode_mtp_fused (Gemma assistant) but for the UNIFIED cache — +-// so no mtp_assistant / sched_mtp / kv_iswa. The N fused steps read-only cross-attend the frozen ++// drafts with a single sync, for the UNIFIED cache — ++// so no mtp_assistant / kv_iswa involvement. The N fused steps read-only cross-attend the frozen + // prefix (build_attn_readonly_nextn, Option A); recurrence is carried by the on-device hidden chain. + // GREEDY-ONLY: no p_min early-stop (the caller reconciles draft length). See + // docs/features/fused_nextn_mtp.md. Returns 0 on success, negative on error. +@@ -3062,7 +2484,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + return -2; + } + // NextN self-spec targets are full-attention → unified cache. (iSWA NextN is not a thing; +- // the Gemma assistant path is decode_mtp_fused.) ++ // the Gemma assistant runs as a dual context via draft_mtp, not through this driver.) + auto * kv = dynamic_cast(memory.get()); + if (!kv) { + LLAMA_LOG_ERROR("%s: fused NextN requires a unified llama_kv_cache\n", __func__); +@@ -3133,7 +2555,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + } + + // Build/compute the fused N-step graph on the NORMAL sched (graph_params sets n_mtp_steps=N for +- // DECODER_MTP). Inlined like process_ubatch_mtp — deliberately NOT process_ubatch, whose apply() ++ // DECODER_MTP). Inlined graph processing — deliberately NOT process_ubatch, whose apply() + // would overwrite the pmax prefix cell. + mtp_fused_steps = n_steps; + +@@ -3187,7 +2609,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + } + res->set_inputs(&ub); + +- // Compute on the dedicated sched (mirror graph_compute_mtp's CPU-threadpool dance). ++ // Compute on the dedicated sched (with the CPU-threadpool attach/set dance). + if (backend_cpu != nullptr) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_tp_fn = (decltype(ggml_backend_cpu_set_threadpool) *) +@@ -3227,224 +2649,6 @@ int32_t llama_context::decode_mtp_fused_nextn( + return 0; + } + +-// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one +-// in-flight request per context (returns -7 if a prior request was not yet waited). +-int32_t llama_context::decode_mtp_async( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- const float * h_prev, +- int32_t n_steps) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0 || !h_prev || n_steps <= 0) { +- LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", +- __func__, n_bb, (const void *) h_prev, n_steps); +- return -4; +- } +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- if (!mtp_worker.joinable()) { +- mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); +- } +- +- { +- std::unique_lock lk(mtp_mu); +- if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { +- LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", +- __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, +- (int) mtp_completed.has_value()); +- return -7; +- } +- mtp_request req; +- req.seq_id = seq_id; +- req.attn_pos = attn_pos; +- req.last_token = last_token; +- req.n_steps = n_steps; +- req.h_prev.assign(h_prev, h_prev + n_bb); +- mtp_pending = std::move(req); +- } +- mtp_cv_request.notify_one(); +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last +-// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). +-int32_t llama_context::decode_mtp_wait( +- llama_token * out_drafts, +- float * out_h_prev_last) { +- std::unique_lock lk(mtp_mu); +- mtp_cv_response.wait(lk, [this] { +- return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); +- }); +- if (!mtp_completed.has_value()) { +- LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); +- return -7; +- } +- mtp_response resp = std::move(*mtp_completed); +- mtp_completed.reset(); +- lk.unlock(); +- +- if (resp.status != 0) { +- return resp.status; +- } +- if (out_drafts && !resp.drafts.empty()) { +- std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); +- } +- if (out_h_prev_last && !resp.h_prev_last.empty()) { +- std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); +- } +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's +-// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the +-// single ubatch row MUST be flagged output — a 0 here reintroduces the null-out-ids abort +-// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. +-int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(req.seq_id); +- data->seq_idx[(size_t) req.seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = req.seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 +- +- std::vector h(req.h_prev); +- llama_token last_token = req.last_token; +- resp.drafts.assign((size_t) req.n_steps, 0); +- +- for (int32_t k = 0; k < req.n_steps; ++k) { +- data->token[0] = last_token; +- data->pos[0] = req.attn_pos + 1 + (llama_pos) k; +- std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); +- int32_t best_i32 = 0; +- ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); +- last_token = (llama_token) best_i32; +- resp.drafts[(size_t) k] = last_token; +- +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); +- } +- +- resp.h_prev_last = std::move(h); +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on +-// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. +-void llama_context::mtp_worker_loop() { +- for (;;) { +- mtp_request req; +- { +- std::unique_lock lk(mtp_mu); +- mtp_cv_request.wait(lk, [this] { +- return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); +- }); +- if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { +- return; +- } +- req = std::move(*mtp_pending); +- mtp_pending.reset(); +- mtp_in_flight = true; +- } +- +- mtp_response resp; +- resp.status = decode_mtp_run(req, resp); +- +- { +- std::lock_guard lk(mtp_mu); +- mtp_in_flight = false; +- mtp_completed = std::move(resp); +- } +- mtp_cv_response.notify_one(); +- } +-} +- +-// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV +-// read retired), then sync sched_mtp — WITHOUT consuming mtp_completed (the spec loop's _wait +-// still needs the drafts). Called at decode() top before any KV mutation. Inert when no +-// assistant is loaded or nothing is in flight (drained==false ⇒ no extra sched sync). +-void llama_context::mtp_drain_before_mutate() { +- if (!model.mtp_assistant) { +- return; +- } +- bool drained = false; +- { +- std::unique_lock lk(mtp_mu); +- if (mtp_pending.has_value() || mtp_in_flight) { +- mtp_cv_response.wait(lk, [this] { +- return !mtp_in_flight && !mtp_pending.has_value(); +- }); +- drained = true; +- } +- } +- if (drained && sched_mtp) { +- ggml_backend_sched_synchronize(sched_mtp.get()); +- } +-} +- + llm_graph_params llama_context::graph_params( + llm_graph_result * res, + const llama_ubatch & ubatch, +@@ -5177,24 +4381,6 @@ int32_t llama_decode( + return ret; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-int32_t llama_decode_mtp( +- llama_context * ctx, +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- if (!ctx) { +- LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); +- return -1; +- } +- return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); +-} +- + int32_t llama_decode_mtp_fused_nextn( + llama_context * ctx, + llama_seq_id seq_id, +diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h +index b6b0efb..32034fd 100644 +--- a/llama.cpp/src/llama-context.h ++++ b/llama.cpp/src/llama-context.h +@@ -147,48 +147,8 @@ struct llama_context { + llama_memory_context_i * mctx, + ggml_status & ret); + +- // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. +- llm_graph_params graph_params_mtp( +- llm_graph_result * res, +- const llama_ubatch & ubatch, +- const llama_memory_context_i * mctx) const; +- +- int32_t decode_mtp( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- +- int32_t decode_mtp_sync( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- +- // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps +- // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft +- // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced +- // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits +- // verification path stays on decode_mtp_sync. +- int32_t decode_mtp_fused( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_h_prev_last); +- + // opencoti fused-NextN (#590): fused N-step greedy draft on a DECODER_MTP (Qwen NextN) draft +- // context using the unified cache. Twin of decode_mtp_fused minus mtp_assistant/sched_mtp/iSWA. ++ // context using the unified cache (the only in-context MTP draft driver). + int32_t decode_mtp_fused_nextn( + llama_seq_id seq_id, + llama_pos attn_pos, +@@ -198,25 +158,6 @@ struct llama_context { + llama_token * out_drafts, + float * out_h_prev_last); + +- // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync +- // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams +- // no per-step logits); otherwise it submits via decode_mtp_async and blocks in +- // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the +- // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). +- int32_t decode_mtp_async( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- const float * h_prev, +- int32_t n_steps); +- int32_t decode_mtp_wait( +- llama_token * out_drafts, +- float * out_h_prev_last); +- // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before +- // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP +- // assistant is loaded or nothing is in flight (the common case). +- void mtp_drain_before_mutate(); +- + int encode(const llama_batch & batch_inp); + int decode(const llama_batch & batch_inp); + +@@ -333,14 +274,6 @@ private: + const llama_memory_context_i * mctx, + llm_graph_type gtype) const; + +- // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. +- bool ensure_sched_mtp(); +- llm_graph_result * process_ubatch_mtp( +- const llama_ubatch & ubatch, +- llama_memory_context_i * mctx, +- ggml_status & ret); +- ggml_status graph_compute_mtp(ggml_cgraph * gf); +- + llm_graph_cb graph_get_cb() const; + + // TODO: read/write lora adapters and cvec +@@ -440,58 +373,22 @@ private: + llm_graph_result_ptr gf_res_prev; + llm_graph_result_ptr gf_res_reserve; + +- // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be +- // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable +- // single-token MTP graph result across steps of one decode_mtp call. +- ggml_backend_sched_ptr sched_mtp; +- llm_graph_result_ptr gf_res_prev_mtp; +- + // opencoti #590/bug-858: DEDICATED scheduler + result for the fused NextN self-spec draft + // (decode_mtp_fused_nextn). It must NOT share the main sched/gf_res_prev with target decode: + // every interleaved target decode RESETS the main sched (reallocating compute buffers) and + // gf_res_prev, so a fused graph parked there can never satisfy can_reuse AND its tensor memory is + // clobbered → 100% rebuild + CUDA-recapture per draft round (measured: build 0.5ms + comp-launch +- // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched (like the Gemma assistant's +- // sched_mtp, but ensure_sched_mtp is assistant-only: needs mtp_assistant + iswa cache) isolates the ++ // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched isolates the + // fused graph so can_reuse compares fused-vs-fused (stable within a 256-cell n_kv bucket) and the + // ggml graph + CUDA capture persist across rounds. NextN and Gemma-assistant are mutually exclusive. + ggml_backend_sched_ptr sched_nextn; + llm_graph_result_ptr gf_res_prev_nextn; + int32_t nextn_reserved_steps = 0; + +- // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for +- // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it +- // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so +- // ensure_sched_mtp only re-reserves when a larger fused block is requested. ++ // opencoti #590 fused NextN: number of draft steps to unroll into the MTP graph for the ++ // NEXT build (decode_mtp_fused_nextn sets it to n_steps around its build; the normal ++ // decode/reserve paths leave it at 1 so graph_params stays single-step). + int32_t mtp_fused_steps = 1; +- int32_t mtp_reserved_steps = 0; +- +- // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. +- // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp +- // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync +- // facade), so the worker never races the target's sched. The destructor joins the worker. +- struct mtp_request { +- llama_seq_id seq_id = 0; +- llama_pos attn_pos = 0; +- llama_token last_token = 0; +- std::vector h_prev; +- int32_t n_steps = 0; +- }; +- struct mtp_response { +- int32_t status = 0; +- std::vector drafts; +- std::vector h_prev_last; +- }; +- std::thread mtp_worker; +- std::atomic mtp_worker_stop{false}; +- std::mutex mtp_mu; +- std::condition_variable mtp_cv_request; +- std::condition_variable mtp_cv_response; +- std::optional mtp_pending; // submitted, not yet picked up +- bool mtp_in_flight = false; // worker is processing +- std::optional mtp_completed; // worker finished, awaiting _wait +- int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop +- void mtp_worker_loop(); + + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_ptr buf_output; +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index e0d712e..6fc9679 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -124,7 +124,7 @@ void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { + ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); + + // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] +- // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. ++ // (= attn_pos + 1 + k, filled by decode_mtp_fused_nextn). Single-step path leaves this empty. + // bug-870: mrope archs (qwen35) size each step's pos tensor I32[4] — fill the M-RoPE + // text-token layout [p,p,p,0] (mirrors llm_graph_input_pos::set_input); standard rope = [p]. + for (size_t k = 0; k < inp_pos_steps.size(); ++k) { +@@ -570,7 +570,7 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { + void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { + // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently — + // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP +- // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but ++ // dual-ctx assistant draft graph shares the target KV through mem_other, but + // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) + // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the + // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP +@@ -2999,95 +2999,9 @@ ggml_tensor * llm_graph_context::build_attn( + return cur; + } + +-// opencoti F5 M6-S4 mtp +-ggml_tensor * llm_graph_context::build_attn_mtp( +- llm_graph_input_attn_kv_iswa * inp, +- ggml_tensor * wo, +- ggml_tensor * wo_b, +- ggml_tensor * q_cur, +- ggml_tensor * kq_b, +- ggml_tensor * sinks, +- ggml_tensor * v_mla, +- float kq_scale, +- int il_mtp, +- int32_t il_kv_tgt, +- bool read_from_swa_kv, +- int64_t kv_embd_head_v, +- int64_t kv_n_head_v, +- bool use_k_as_v) const { +- auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; +- auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; +- +- if (k_rot) { +- q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); +- } +- if (v_rot) { +- // V-only rotation is applied after MHA in the standard path; no V write here. +- } +- +- ggml_build_forward_expand(gf, q_cur); +- +- const auto * mctx_iswa = inp->mctx; +- const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); +- +- const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); +- +- ggml_tensor * q = q_cur; +- ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); +- ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); +- +- if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0 +- || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) +- // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the +- // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp +- // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at +- // head_dim∈{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 +- // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired +- // inverse on the output → wrong logits (graph still *built*, which is why P2's inert build +- // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity +- // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. +- if (q->ne[0] % 128 != 0) { +- const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; +- q = ggml_pad(ctx0, q, pad, 0, 0, 0); +- } +- if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } +- } +- +- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); +- cb(cur, "kqv_out_mtp", il_mtp); +- +- if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0 +- || v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) +- const int64_t orig_v_head = kv_embd_head_v; +- const int64_t padded_v_head = v->ne[0]; +- if (padded_v_head != orig_v_head) { +- const int64_t n_head_v = kv_n_head_v; +- const int64_t n_tokens_cur = cur->ne[1]; +- cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); +- cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, +- cur->nb[1], cur->nb[2], 0); +- cur = ggml_cont(ctx0, cur); +- cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); +- } +- } +- +- if (v_rot) { +- cur = ggml_mul_mat_aux(ctx0, cur, v_rot); +- } +- +- if (wo) { +- cur = build_lora_mm(wo, cur); +- cb(cur, "mtp_wo_out", il_mtp); +- } +- if (wo_b) { +- cur = ggml_add(ctx0, cur, wo_b); +- } +- +- return cur; +-} + + // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the frozen prefix +-// KV on the UNIFIED cache. Twin of build_attn_mtp for the non-iSWA llm_graph_input_attn_kv used by ++// KV on the UNIFIED cache via the non-iSWA llm_graph_input_attn_kv used by + // Qwen NextN drafters. Reads mctx->get_k/get_v(il) with the input's kq_mask — which exposes the + // full prefix 0..pmax because get_n_kv reports the cache's used extent, not the mtp_slot_info cell + // count. Writes NO draft KV; recurrence is carried by the hidden-state chain. wo is applied by the +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +index d3a3499..daa61b0 100644 +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -595,7 +595,7 @@ struct llm_graph_params { + // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. + // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone + // into the next step with no host round-trip / per-step GPU sync (the throughput fix). +- // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. ++ // Read by the fused NextN builders; default 1 keeps every non-MTP construction unchanged. + int32_t n_mtp_steps = 1; + + // return true if the "other" params would result in a graph with the same topology as with the current params +@@ -1021,28 +1021,8 @@ struct llm_graph_context { + int il, + bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) + +- // opencoti F5 M6-S4 mtp +- // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). +- // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). +- // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). +- ggml_tensor * build_attn_mtp( +- llm_graph_input_attn_kv_iswa * inp, +- ggml_tensor * wo, +- ggml_tensor * wo_b, +- ggml_tensor * q_cur, +- ggml_tensor * kq_b, +- ggml_tensor * sinks, +- ggml_tensor * v_mla, +- float kq_scale, +- int il_mtp, +- int32_t il_kv_tgt, +- bool read_from_swa_kv, +- int64_t kv_embd_head_v, +- int64_t kv_n_head_v, +- bool use_k_as_v) const; +- + // opencoti fused-NextN (Option A): read-only cross-attention into the frozen prefix KV on the +- // UNIFIED cache. Twin of build_attn_mtp for llm_graph_input_attn_kv (non-iSWA). Reads ++ // UNIFIED cache (llm_graph_input_attn_kv). Reads + // mctx->get_k/get_v(il) with the input's kq_mask (which exposes 0..pmax via get_n_kv's used + // extent), runs build_attn_mha, writes NO draft KV. wo is applied by the caller. + // See docs/features/fused_nextn_mtp.md. +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index 9628df1..f2e95a0 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -247,21 +247,6 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_full() { + return std::make_unique(this); + } + +-// opencoti F5 M6-S4 mtp +-llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { +- llama_kv_cache::slot_info_vec_t sinfos_base; +- llama_kv_cache::slot_info_vec_t sinfos_swa; +- +- sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); +- sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); +- +- std::vector ubatches; +- ubatches.push_back(std::move(ubatch)); +- +- return std::make_unique( +- this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); +-} +- + llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); + } +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 06fd1ae..a7eec3d 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -77,8 +77,6 @@ public: + + llama_memory_context_ptr init_full() override; + +- // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). +- llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 4416d38..71bf0d5 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -5439,7 +5439,7 @@ llama_kv_cache_context::llama_kv_cache_context( + llama_kv_cache::slot_info_vec_t sinfos, + std::vector ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { + // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, +- // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from ++ // which builds its graph inline and never calls apply()) get a valid mask/K/V shape from + // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a + // view with a garbage dim-2 (observed ne[2]=2956782432 → ggml.c view-bounds assert). For + // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. +diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp +index cab4fa8..64ea519 100644 +--- a/llama.cpp/src/models/gemma4-assistant.cpp ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -4,380 +4,10 @@ + #include + #include + +-static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { +- p.arch = mtp_model.arch; +- p.hparams = mtp_model.hparams; +- p.gtype = LLM_GRAPH_TYPE_MTP; +- return p; +-} +- +-// Last layer in [range_start, range_end) whose attention type matches want_swa. +-static int32_t gemma4_mtp_kv_layer_last_in_range( +- const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { +- int32_t best = -1; +- if (range_start < 0) { +- range_start = 0; +- } +- if (range_end > (int32_t) tgt.n_layer) { +- range_end = (int32_t) tgt.n_layer; +- } +- for (int32_t il = range_start; il < range_end; ++il) { +- if (tgt.is_swa((uint32_t) il) == want_swa) { +- best = il; +- } +- } +- return best; +-} +- +-// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> +-// (post-projected backbone hidden, vocabulary logits, argmax token id). +-// +-// Used by both the single-step path (n_mtp_steps==1) and the chained path +-// (n_mtp_steps>1, called n_mtp_steps times within the same graph). +-// +-// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position +-// for this step's RoPE (the cross-attn mask is shared across steps because the +-// target's KV cache only contains positions ≤ attn_pos and all step positions +-// are > attn_pos, so causal/SWA admit all target cells uniformly). +-// +-// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too — required for greedy match with verify). +-// `out_argmax` (I32 [1]): greedy token id on-device. +-static void gemma4_mtp_build_one_step( +- const llm_graph_context & gctx, +- const llama_model & target, +- const llama_model & mtp, +- llm_graph_input_attn_kv_iswa * inp_attn, +- ggml_tensor * tok_step, // I32 [1] +- ggml_tensor * h_step, // F32 [n_bb, 1] +- ggml_tensor * pos_step, // I32 [1] +- ggml_tensor ** out_logits, // F32 [n_vocab, 1] +- ggml_tensor ** out_h_post, // F32 [n_bb, 1] +- ggml_tensor ** out_argmax) { // I32 [1] +- auto * ctx0 = gctx.ctx0; +- auto * gf = gctx.gf; +- const auto & hparams = gctx.hparams; +- const auto & cparams = gctx.cparams; +- const int n_layer = (int) gctx.n_layer; +- const int n_tokens = (int) gctx.n_tokens; +- const int n_ctx_orig = (int) gctx.n_ctx_orig; +- const int rope_type = gctx.rope_type; +- const float ext_factor = gctx.ext_factor; +- const float attn_factor = gctx.attn_factor; +- const float beta_fast = gctx.beta_fast; +- const float beta_slow = gctx.beta_slow; +- auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; +- +- const float tok_scale = sqrtf((float) target.hparams.n_embd); +- +- ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); +- cb(tok_e, "mtp_tgt_tok_embd", -1); +- +- // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). +- // Use target n_embd so Edge / non-Edge targets match the main forward. +- tok_e = ggml_scale(ctx0, tok_e, tok_scale); +- cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); +- +- ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); +- cb(inp_cat, "mtp_concat", -1); +- +- ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); +- cb(inpL, "mtp_pre_proj_out", -1); +- +- ggml_build_forward_expand(gf, inpL); +- +- ggml_tensor * cur = nullptr; +- +- for (int il = 0; il < n_layer; ++il) { +- const int64_t n_embd_head = hparams.n_embd_head_k(il); +- GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); +- +- const int64_t n_head = hparams.n_head(il); +- +- const float freq_base_l = mtp.get_rope_freq_base(cparams, il); +- const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); +- const int n_rot_l = hparams.n_rot(il); +- +- cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "attn_norm", il); +- +- ggml_tensor * freq_factors = nullptr; +- if (!hparams.is_swa(il)) { +- freq_factors = mtp.layers[il].rope_freqs; +- } +- +- ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); +- cb(Qcur, "Qcur", il); +- +- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); +- +- Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); +- cb(Qcur, "Qcur_normed", il); +- +- Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, +- ext_factor, attn_factor, beta_fast, beta_slow); +- cb(Qcur, "Qcur_pos", il); +- +- const bool read_swa = hparams.is_swa(il); +- +- const int32_t n_tgt = (int32_t) target.hparams.n_layer; +- +- // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads +- // ONE shared KV per attention type from the target — the LAST layer of that type. +- // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py +- // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} +- const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); +- +- GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); +- +- // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and +- // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the +- // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must +- // ALWAYS fetch V from the cache — not reuse the post-RoPE K tensor. +- const bool use_k_as_v = false; +- +- const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); +- const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); +- +- cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, +- hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); +- +- cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "attn_post_norm", il); +- +- ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); +- cb(attn_out, "attn_out", il); +- +- GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); +- +- cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "ffn_norm", il); +- +- cur = gctx.build_ffn(cur, +- mtp.layers[il].ffn_up, nullptr, nullptr, +- mtp.layers[il].ffn_gate, nullptr, nullptr, +- mtp.layers[il].ffn_down, nullptr, nullptr, +- nullptr, +- LLM_FFN_GELU, LLM_FFN_PAR, il); +- cb(cur, "ffn_out", il); +- +- cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); +- cb(cur, "ffn_post_norm", il); +- +- cur = ggml_add(ctx0, cur, attn_out); +- +- if (mtp.layers[il].out_scale) { +- cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); +- cb(cur, "out_scaled", il); +- } +- +- cur = gctx.build_cvec(cur, il); +- cb(cur, "l_out", il); +- +- inpL = cur; +- } +- +- cur = inpL; +- +- cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); +- cb(cur, "result_norm", -1); +- +- ggml_tensor * h_inner = cur; +- +- ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); +- cb(backbone, "mtp_post_proj_out", -1); +- +- const int64_t n_vocab = mtp.tok_embd->ne[1]; +- const int64_t n_tokens_mtp = h_inner->ne[1]; +- +- if (mtp.hparams.use_ordered_embeddings) { +- // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full +- // [n_vocab] row then argmax — matches masked full-vocab greedy (sparse-only argmax broke server accept). +- GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); +- GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); +- const uint32_t n_c = mtp.hparams.n_centroids; +- const uint32_t top_k = mtp.hparams.centroid_top_k; +- GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); +- GGML_ASSERT(n_vocab % (int64_t) n_c == 0); +- const int64_t vsc = n_vocab / (int64_t) n_c; +- +- ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); +- cb(centroid_logits, "mtp_centroid_logits", -1); +- +- ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); +- cb(topk_idx, "mtp_centroid_topk_idx", -1); +- +- const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); +- ggml_tensor * ordering = ggml_view_2d( +- ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); +- cb(ordering, "mtp_token_ordering_view", -1); +- +- ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); +- cb(sel_ids, "mtp_selected_token_ids", -1); +- +- const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; +- ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); +- cb(flat_ids, "mtp_selected_token_ids_flat", -1); +- +- ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); +- cb(sel_emb, "mtp_selected_embd", -1); +- +- ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); +- cb(sel_logits, "mtp_selected_logits", -1); +- ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); +- +- ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); +- logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); +- cb(logits_full, "mtp_logits_masked_base", -1); +- +- ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); +- ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); +- cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); +- cb(cur, "mtp_logits_scatter_view", -1); +- cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); +- cb(cur, "mtp_logits_full", -1); +- } else { +- cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); +- cb(cur, "result_output_dense", -1); +- } +- +- if (hparams.f_final_logit_softcapping) { +- cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); +- cur = ggml_tanh(ctx0, cur); +- cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); +- } +- +- cb(cur, "result_output", -1); +- +- // Greedy argmax on-device: I32 [1] token index into the vocabulary row. +- ggml_tensor * arg = ggml_argmax(ctx0, cur); +- cb(arg, "result_argmax", -1); +- +- *out_logits = cur; +- *out_h_post = backbone; +- *out_argmax = arg; +-} +- +-llm_build_gemma4_mtp::llm_build_gemma4_mtp( +- const llama_model & target_model, +- const llama_model & mtp_model, +- const llm_graph_params & params) : +- llm_graph_context(graph_params_for_mtp(params, mtp_model)), +- target(target_model), +- mtp(mtp_model) { +- const int64_t n_bb = mtp.hparams.n_embd_out_impl; +- GGML_ASSERT(n_bb > 0); +- GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); +- +- // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 +- // (fused path) chain from the previous step's on-device argmax + post-projection, so only +- // step 0 reads token/h from the host. +- ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); +- ggml_set_input(inp_tok); +- cb(inp_tok, "mtp_inp_last_token", -1); +- +- ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); +- ggml_set_input(inp_h); +- cb(inp_h, "mtp_inp_h_prev", -1); +- +- // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into +- // ONE graph (no per-step host round-trip / GPU sync — the throughput fix). n_steps==1 is the +- // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). +- const int32_t n_steps = std::max(params.n_mtp_steps, 1); +- +- auto * inp_attn = build_attn_inp_kv_iswa(); +- +- if (n_steps <= 1) { +- { +- auto inp_wrap = std::make_unique(); +- inp_wrap->inp_last_token = inp_tok; +- inp_wrap->inp_h_prev = inp_h; +- res->add_input(std::move(inp_wrap)); +- } +- +- ggml_tensor * inp_pos = build_inp_pos(); +- +- ggml_tensor * logits = nullptr; +- ggml_tensor * h_post = nullptr; +- ggml_tensor * arg = nullptr; +- gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, +- inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); +- +- res->t_embd = h_post; +- res->t_logits = logits; +- res->t_argmax = arg; +- +- ggml_build_forward_expand(gf, arg); +- ggml_build_forward_expand(gf, h_post); +- return; +- } +- +- // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is +- // shared across steps — see gemma4_mtp_build_one_step's contract). The chain ties step k+1's +- // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. +- auto inp_wrap = std::make_unique(); +- inp_wrap->inp_last_token = inp_tok; +- inp_wrap->inp_h_prev = inp_h; +- inp_wrap->inp_pos_steps.reserve(n_steps); +- +- std::vector pos_steps; +- pos_steps.reserve(n_steps); +- for (int32_t k = 0; k < n_steps; ++k) { +- ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); +- ggml_set_input(p); +- cb(p, "mtp_inp_pos_step", k); +- pos_steps.push_back(p); +- inp_wrap->inp_pos_steps.push_back(p); +- } +- res->add_input(std::move(inp_wrap)); +- +- ggml_tensor * tok_k = inp_tok; +- ggml_tensor * h_k = inp_h; +- +- std::vector step_args; +- step_args.reserve(n_steps); +- ggml_tensor * last_logits = nullptr; +- ggml_tensor * last_h_post = nullptr; +- +- for (int32_t k = 0; k < n_steps; ++k) { +- ggml_tensor * logits_k = nullptr; +- ggml_tensor * h_post_k = nullptr; +- ggml_tensor * arg_k = nullptr; +- gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, +- tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); +- +- step_args.push_back(arg_k); +- last_logits = logits_k; +- last_h_post = h_post_k; +- +- // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the +- // target vocab — already used as a get_rows index inside build_one_step), next step's +- // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. +- tok_k = arg_k; +- h_k = h_post_k; +- } +- +- // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 +- // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend — a +- // negligible side-consumer that does NOT touch the on-device get_rows chain above. +- ggml_tensor * all_args = step_args[0]; +- for (int32_t k = 1; k < n_steps; ++k) { +- all_args = ggml_concat(ctx0, all_args, step_args[k], 0); +- } +- cb(all_args, "mtp_fused_argmax", -1); +- +- res->t_argmax = all_args; // I32 [n_steps] — the drafted token block +- res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed +- res->t_logits = last_logits; // unused on the live path (out_logits == NULL) +- +- ggml_build_forward_expand(gf, all_args); +- ggml_build_forward_expand(gf, last_h_post); +-} +- + // opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via + // llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path +-// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the +-// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 +-// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. ++// (llama_model_mapping -> this class -> load_arch_hparams/tensors). The assistant is never a ++// primary model; build_arch_graph builds the dual-context drafter graph (ctx_other required). + void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); +@@ -443,7 +73,7 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + // + // opencoti bug-858 (dual-context MTP): pre_projection is classified LLM_TENSOR_LAYER_INPUT, so the + // plain create_tensor lands it on the CPU buft. The single-context facade tolerates that (it runs on +- // the target's sched_mtp, which has the CPU backend registered), but a REAL draft context ctx_dft ++ // the target's scheduler, which has the CPU backend registered), but a REAL draft context ctx_dft + // (llama_init_from_model over the nested assistant) registers only the assistant's *offloaded* (CUDA) + // weight buffers — a CPU-resident weight then reads back uninitialized → the whole drafter forward is + // NaN → 0% accept. Force it onto the output-head (GPU-when-offloaded) buft, exactly like tok_embd +@@ -502,14 +132,12 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + // wk/wv): each attention layer reads the TARGET's shared K/V *in place* via the standard iSWA + // build_attn (Qcur only, null k_cur/v_cur -> no store). The draft context's create_memory (A3) + // shares the target's KV cells and aliases the target's last full (n_layer-1) / SWA (n_layer-2) +-// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V — +-// which is exactly what the single-context path did explicitly via build_attn_mtp(..., il_kv). ++// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V. + // The input token embedding + backbone hidden come from the target model via ctx_other. ++// (The retired single-context facade did this explicitly via a build_attn_mtp cross-read; removed ++// in #611 once this dual-context path proved parity — see UPSTREAM_SYNC.md.) + // +-// This is DISTINCT from the single-context llm_build_gemma4_mtp (built by the gemma4 TARGET's +-// build_arch_graph, gemma4.cpp) which is kept fully intact as the fallback drafter. +-// +-// Faithful to upstream b9859 (and unlike our single-context build): dense LM head over tok_embd ++// Faithful to upstream b9859: dense LM head over tok_embd + // (no ordered-embeddings/centroid path — b9859 loads centroids but ignores them in the graph), + // no final-logit softcapping (monotonic -> argmax-invariant), and no control-vector application + // (inert without a control vector). See docs/evaluations/mtp.md. +@@ -655,8 +283,6 @@ struct llm_build_gemma4_assistant_dual : public llm_graph_context { + std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { + // opencoti bug-858 dual-context MTP (A6): build the gemma4-assistant drafter graph. This model + // is KV-less and only valid as an MTP draft context (ctx_dft) created FROM it with +- // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. The single-context +- // engine (llm_build_gemma4_mtp) is built by the gemma4 TARGET's build_arch_graph instead and +- // never reaches here; it remains the fallback drafter. ++ // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. + return std::make_unique(*this, params); + } +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +index bdf5990..464e5f5 100644 +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -132,14 +132,6 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { + } + + std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { +- // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), +- // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF +- // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ +- // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. +- if (params.gtype == LLM_GRAPH_TYPE_MTP) { +- GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); +- return std::make_unique(*this, *mtp_assistant, params); +- } + return std::make_unique(*this, params); + } + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +index deb2566..6efae4a 100644 +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -809,19 +809,10 @@ struct llama_model_gemma4 : public llama_model_base { + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; + }; + +-// opencoti F5 M6-S4 mtp +-// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. +-struct llm_build_gemma4_mtp : public llm_graph_context { +- const llama_model & target; +- const llama_model & mtp; +- +- llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); +-}; +- + // opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant — loaded INTO a gemma4 target via + // llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own +-// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the +-// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). ++// hparams/tensors; build_arch_graph builds the dual-context drafter graph (requires ++// cparams.ctx_other = the Gemma 4 target, bug-858). + struct llama_model_gemma4_assistant : public llama_model_base { + llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; +diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp +index 3a21d69..827779e 100644 +--- a/llama.cpp/src/models/qwen35.cpp ++++ b/llama.cpp/src/models/qwen35.cpp +@@ -628,7 +628,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the + // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used + // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. +- // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. + cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); + } else { + Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +@@ -700,8 +700,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + return; + } + +- // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant +- // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver ++ // sets n_mtp_steps>1 (S3). One + // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 + // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] + // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, +diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp +index 0eb2871..af9ea1e 100644 +--- a/llama.cpp/src/models/qwen35moe.cpp ++++ b/llama.cpp/src/models/qwen35moe.cpp +@@ -680,7 +680,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the + // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used + // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. +- // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. + cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); + } else { + Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +@@ -784,8 +784,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + return; + } + +- // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant +- // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver ++ // sets n_mtp_steps>1 (S3). One + // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 + // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] + // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 34b50b1..04207b2 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -918,25 +918,21 @@ private: + + add_bos_token = llama_vocab_get_add_bos(vocab); + +- // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so +- // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file — +- // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at +- // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from +- // both native branches (separate-model draft below; NextN MTP-ctx-against-target further +- // down). Detect it via the enabled types vector, load into the target, and leave +- // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's +- // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). ++ // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4-assistant draft head is a SEPARATE GGUF (so ++ // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file, ++ // then runs as a REAL draft context (ctx_dft) whose cparams.ctx_other = ctx_tgt shares the ++ // target's KV (bug-858 dual-context, upstream b9859 shape). This is distinct from both ++ // native branches (separate-model draft below; NextN MTP-ctx-against-target further down). ++ // Detect it via the enabled types vector and load into the target first. + const bool spec_assistant = std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); + if (params_base.speculative.has_dft() && spec_assistant) { +- // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target +- // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the +- // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout +- // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. +- // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 +- // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it +- // explicitly and fail clean at boot instead of crashing mid-request. ++ // opencoti F5 M6-S4 mtp (#485/#611): with --parallel >1 WITHOUT --kv-unified the KV is ++ // per-stream-split (patch 0031); the assistant's shared-KV aliasing (mem_other) assumes the ++ // unified per-layer tensor layout, so per-stream splits are not supported. The unified ++ // cache IS handled (proven on the facade at --parallel 2 --kv-unified — the PolyKV config, ++ // which forces --kv-unified). Require it explicitly and fail clean at boot. + if (params_base.n_parallel > 1 && !params_base.kv_unified) { + SRV_ERR("%s", + "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " +@@ -965,21 +961,19 @@ private: + return false; + } + +- // opencoti bug-858 dual-context MTP (A5): optionally create a REAL draft context FROM the +- // nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 ++ // opencoti bug-858 dual-context MTP (A5, default since #611): create a REAL draft context ++ // FROM the nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 + // dual-context). The assistant is KV-less: its create_memory (A3) shares the target's KV + // cells + aliases the last full/SWA layer, and its graph (A6) reads the target's tok_embd + // + shared K/V via ctx_other; the shared-KV draft_mtp driver (is_mem_shared) then runs it. +- // Gated by OPENCOTI_MTP_DUAL_CTX so the default keeps the proven single-context in-target +- // path (ctx_dft == nullptr, llama_decode_mtp) byte-identical. +- const char * mtp_dual_env = getenv("OPENCOTI_MTP_DUAL_CTX"); +- const bool mtp_dual_ctx = mtp_dual_env && mtp_dual_env[0] && mtp_dual_env[0] != '0'; +- if (mtp_dual_ctx) { ++ // This is the ONLY assistant execution path — the single-context in-target facade ++ // (llama_decode_mtp) was removed in #611 after the port proved accept parity. ++ { + // llama_init_from_model wants a mutable model*; the nested assistant is owned mutably + // by the target (unique_ptr), so the const_cast is safe. + llama_model * assistant = const_cast(llama_model_get_mtp_assistant(model_tgt)); + if (assistant == nullptr) { +- SRV_ERR("%s", "MTP dual-context requested but the assistant is not loaded into the target\n"); ++ SRV_ERR("%s", "MTP assistant is not loaded into the target\n"); + return false; + } + +@@ -1002,11 +996,6 @@ private: + params_base.speculative.draft.ctx_tgt = ctx_tgt; + params_base.speculative.draft.ctx_dft = ctx_dft.get(); + SRV_INF("%s", "MTP assistant dual-context draft created (ctx_other=target, shared KV)\n"); +- } else { +- // No separate draft context: the assistant decodes on ctx_tgt via llama_decode_mtp. +- params_base.speculative.draft.ctx_tgt = ctx_tgt; +- params_base.speculative.draft.ctx_dft = nullptr; +- SRV_INF("%s", "MTP assistant loaded into target (single-context)\n"); + } + } else if (params_base.speculative.has_dft()) { + // TODO speculative: move to common/speculative.cpp? diff --git a/patches/0097-native-elf-cmake.patch b/patches/0097-native-elf-cmake.patch new file mode 100644 index 0000000000000000000000000000000000000000..cbaf14d2af14dd432401537a8b8b64118fbf37d0 --- /dev/null +++ b/patches/0097-native-elf-cmake.patch @@ -0,0 +1,140 @@ +opencoti patch 0097 — native-ELF CMake lane (#619, secondary artifact of #613) + +Makes the vendored tree buildable as a standard Linux ELF llama-server via +plain cmake + g++/nvcc (rpath libggml-cuda.so; clean gdb/nsys), alongside the +cosmocc Makefile lane which is UNTOUCHED (BUILD.mk lists sources explicitly +and never reads CMakeLists; the new stub TU is CMake-lane-only). + +- 5 CMakeLists surgical hooks: add opencoti TUs absent from upstream source + lists (ggml-turbo-quant.c, ggml-neo-pipeline.cpp, ggml-iqk-flash-attn.cpp + -> ggml-base; dca.cpp -> llama; pcie-profile.cpp -> common; ELF stubs -> + ggml-cpu) + compile the Mozilla-overlay server.cpp with -Dmain=llama_server + (upstream defines llama_server directly; the cosmo lane renames at build). +- NEW ggml/src/ggml-cpu/opencoti-elf-stubs.cpp: return-false stubs for the + cosmocc-bound llamafile/sgemm.cpp CPU fastpath dispatchers (mixmul, + mixmul_needs, mixmul_iqk, fa_* helpers) — every call site falls back to the + upstream generic path (same contract as fa_helpers_unsupported.cpp), so the + ELF build loses only CPU fastpaths, never correctness. + +Build (bs2-proven 2026-07-05): cmake -S llama.cpp -B build-elf -DGGML_CUDA=ON +-DGGML_CUDA_FA_ALL_QUANTS=ON (REQUIRED — turbo/TCQ FA-VEC instances are only +globbed under this flag; cosmo lane always sets it) -DCMAKE_CUDA_ARCHITECTURES=120 +-DCMAKE_BUILD_TYPE=Release -DLLAMA_CURL=OFF -DGGML_NATIVE=OFF. +Gate: A4B -ngl 99 boot, dual-ctx assistant-MTP accept 0.764, 310.2 tps decode. + +--- a/llama.cpp/ggml/src/CMakeLists.txt ++++ b/llama.cpp/ggml/src/CMakeLists.txt +@@ -205,6 +205,9 @@ + ggml-threading.cpp + ggml-threading.h + ggml-quants.c ++ ggml-turbo-quant.c # opencoti-hook: native-elf-cmake (#619) — turbo/TCQ CPU quant, in cosmo BUILD.mk but absent here ++ ggml-neo-pipeline.cpp # opencoti-hook: native-elf-cmake (#619) — NEO pipeline control (pure std) ++ ggml-iqk-flash-attn.cpp # opencoti-hook: native-elf-cmake (#619) — iqk FA enable flag (pure ggml) + ggml-quants.h + gguf.cpp) + +--- a/llama.cpp/src/CMakeLists.txt ++++ b/llama.cpp/src/CMakeLists.txt +@@ -18,6 +18,7 @@ + llama-context.cpp + llama-cparams.cpp + llama-grammar.cpp ++ dca.cpp # opencoti-hook: native-elf-cmake (#619) — DCA (matches neither llama-*.cpp list nor models/ glob) + llama-graph.cpp + llama-hparams.cpp + llama-impl.cpp +--- a/llama.cpp/common/CMakeLists.txt ++++ b/llama.cpp/common/CMakeLists.txt +@@ -66,6 +66,7 @@ + chat.cpp + chat.h + common.cpp ++ pcie-profile.cpp # opencoti-hook: native-elf-cmake (#619) — PCIe bandwidth profile (rolling-KV) + common.h + console.cpp + console.h +--- a/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt ++++ b/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +@@ -52,6 +52,7 @@ + ggml-cpu/vec.cpp + ggml-cpu/ops.h + ggml-cpu/ops.cpp ++ ggml-cpu/opencoti-elf-stubs.cpp # opencoti-hook: native-elf-cmake (#619) — sgemm.cpp dispatcher stubs (cosmo lane uses real sgemm.cpp) + ) + + target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17) +--- a/llama.cpp/tools/server/CMakeLists.txt ++++ b/llama.cpp/tools/server/CMakeLists.txt +@@ -31,6 +31,11 @@ + + set(TARGET llama-server-impl) + ++# opencoti-hook: native-elf-cmake (#619) — the Mozilla overlay's server.cpp defines ++# `int main` (the cosmocc Makefile lane renames it at compile time); upstream ++# defines `llama_server` directly. Rename here so main.cpp's wrapper links. ++set_source_files_properties(server.cpp PROPERTIES COMPILE_DEFINITIONS "main=llama_server") ++ + add_library(${TARGET} + server.cpp + server-http.cpp +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cpu/opencoti-elf-stubs.cpp +@@ -0,0 +1,57 @@ ++// opencoti-hook: native-elf-cmake (#619) — see docs/features/llamafile_build.md ++// ++// CMake(native-ELF)-lane-ONLY stubs for the llamafile/sgemm.cpp CPU fastpath ++// dispatchers. sgemm.cpp includes (cosmocc runtime CPUID/dispatch) ++// and cannot compile under plain g++; every call site treats a false return ++// as "not handled, use the upstream ggml helper" (ggml-cpu.c:1723 mixmul_iqk, ++// ops.cpp:8519/8560 flash-attn helpers — same contract as Mozilla's own ++// fa_helpers_unsupported.cpp), so these stubs are behavior-safe: the native ++// ELF build loses only the iqk/tinyBLAS *CPU* fastpaths, never correctness. ++// The cosmocc Makefile lane never compiles this file (BUILD.mk lists sources ++// explicitly) and keeps the real sgemm.cpp implementations. ++ ++#include ++#include ++ ++struct ggml_compute_params; ++struct ggml_tensor; ++ ++extern "C" { ++ ++// Top-level tinyBLAS MoE mixmul (ggml-cpu.c:1592, GGML_USE_LLAMAFILE): ++// false -> generic mul_mat_id path. _needs sizes its work buffer ++// (ggml-cpu.c:3002): 0 is consistent with mixmul never running. ++bool llamafile_mixmul(const struct ggml_compute_params *, ++ const struct ggml_tensor *, ++ const struct ggml_tensor *, ++ const struct ggml_tensor *, ++ struct ggml_tensor *) { ++ return false; ++} ++ ++size_t llamafile_mixmul_needs(const struct ggml_tensor *, ++ const struct ggml_tensor *, ++ const struct ggml_tensor *) { ++ return 0; ++} ++ ++bool llamafile_mixmul_iqk(long, long, long, int, int, ++ const void *, const void *, float *, ++ long, long, const void *, int, int) { ++ return false; ++} ++ ++bool llamafile_fa_vec_dot_f16(int, float *, const void *, const void *) { ++ return false; ++} ++ ++bool llamafile_fa_fp16_to_fp32_row(const void *, float *, int64_t) { ++ return false; ++} ++ ++bool llamafile_fa_simd_gemm(float *, const float *, const float *, ++ int, int, int) { ++ return false; ++} ++ ++} // extern "C" diff --git a/patches/0098-rolling-kv-lse-decode.patch b/patches/0098-rolling-kv-lse-decode.patch new file mode 100644 index 0000000000000000000000000000000000000000..3e7ed0f799145a3b8b28374f402adc6965810ba4 --- /dev/null +++ b/patches/0098-rolling-kv-lse-decode.patch @@ -0,0 +1,97 @@ +opencoti patch 0098 — rolling-KV: dst_lse emission for head_dim<=256 decode (bug-1843, task #623) + +Extends the Stage-3a LSE channel (opencoti_fattn_dst_lse, drained by +launch_fattn's flash_attn_combine_results / stream-k fixup) to head_dim<=256 +n_q==1 decode at all three streaming-FA call sites (S3b overlap, S3a +single-stream, S2 resident-view). Previously only head_dim>256 armed the +channel; Qwen-class D128 decode fell through to the per-tile +streaming_lse_kernel recompute — a <<>> full key-rescan per +tile that was MEASURED as ~80% of spill-decode wall (bug-1843 A/B on the 3090: +1.14 tps base vs 6.05 tps OPENCOTI_LSE_NOOP, Qwen3-8B-Q8_0 ctx40960 vt14000). + +Mechanism: after the per-tile FA call, `lse_emitted = decode_lse && +opencoti_fattn_dst_lse_written` — the written flag reports whether a finalize +(stream-k fixup or parallel-blocks combine) actually ran; the recompute now +fires only when the dispatch resolved to a no-finalize single-block path. The +combine emits the identical per-row value (kqmax + logf(denominator), row +order h + n_head*(qi + n_q*b)) the recompute produced. The `this_kv > +2*FATTN_KQ_STRIDE` guard is retained everywhere (bug-263 conservatism). +Prefill and D>256 decode are behavior-identical. + +Gates (2026-07-05, 3090, DSO df46dc36): +- perf: spill_base 1.14 -> 6.04 tps (5.3x, at the NOOP ceiling), needle + PRESENT; resident 42.5 tps unchanged; OPENCOTI_LSE_NOOP now speed-inert + (6.02) proving the recompute is structurally off the path. +- correctness: teacher-forced logit-equiv (Qwen3-8B D128, ours-resident REF vs + POSITION_WINDOW 256/16384 cells resident): real_frac=0.0, divergences=0, + frac_full_agree=1.0, mean_tv 0.0068 -> PASS. + +DSO-only (fattn.cu; no header change, no host rebuild needed). + +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -1596,11 +1596,20 @@ + // ntiles_KV=16 at 512, not 2). `>` is kept only because it is no worse + // than `>=`; the real mechanism is unisolated. Real Stage-3c tail tiles + // can be ≤512 at decode → MUST resolve before relying on this path. +- const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ // opencoti-hook: f5-rolling-kv bug-1843 — head_dim<=256 (Qwen-class VEC ++ // decode) now ALSO arms the channel: the parallel-blocks combine emits ++ // the per-row lse (Stage-3a formula in flash_attn_combine_results), and ++ // opencoti_fattn_dst_lse_written reports whether a finalize actually ran ++ // (T87.pD-opt O2 pattern) so the streaming_lse recompute below fires only ++ // when the dispatch resolved to a no-finalize single-block path. The ++ // recompute was measured at ~80% of spill-decode cost (1.14 vs 6.05 tps ++ // A/B via OPENCOTI_LSE_NOOP). ++ const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE); + if (decode_lse) opencoti_fattn_dst_lse = lse_t; + ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ const bool lse_emitted = decode_lse && opencoti_fattn_dst_lse_written; + +- if (!decode_lse) { ++ if (!lse_emitted) { + // opencoti F5 bug-1840 fix (b) DIAGNOSTIC — OPENCOTI_LSE_NOOP skips + // the per-tile streaming_lse recompute (a low-parallelism + // <<>> rescan run once per tile because Qwen/head_dim≤256 +@@ -1727,11 +1736,14 @@ + // (softcap-aware). The `> 2*FATTN_KQ_STRIDE` boundary is NOT a fix for + // bug-263 (UNRESOLVED: decode tile_kv_full==512 corrupts on either lse + // path) — kept only as no-worse-than-`>=`. See .wolf/buglog.json. +- const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ // bug-1843: head_dim<=256 also arms the channel; recompute only when no ++ // finalize ran (opencoti_fattn_dst_lse_written) — see the S3b note above. ++ const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE); + if (decode_lse) opencoti_fattn_dst_lse = lse_t; + ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ const bool lse_emitted = decode_lse && opencoti_fattn_dst_lse_written; + +- if (!decode_lse) { ++ if (!lse_emitted) { + streaming_lse_kernel<<>>( + (const char *)Q->data, sK, + mask ? (const char *)mask->data + (size_t)kv0*mask->nb[0] : nullptr, +@@ -1764,7 +1776,10 @@ + // boundary is NOT a fix for bug-263 (UNRESOLVED: decode tile_kv_full==512 + // corrupts on either lse path) — kept only as no-worse-than-`>=`. See the + // S3b overlap path's full note + .wolf/buglog.json. +- const bool decode_lse = (n_q == 1 && head_dim > 256 && this_kv > 2*FATTN_KQ_STRIDE); ++ // bug-1843: head_dim<=256 also arms the channel; recompute only when no ++ // finalize ran (opencoti_fattn_dst_lse_written) — see the S3b note above. ++ const bool decode_lse = (n_q == 1 && this_kv > 2*FATTN_KQ_STRIDE); ++ bool lse_emitted = false; + if (this_kv > 0) { + // FA over the tile's key-slice → O_t (stock kernel, untouched). + ggml_tensor Kt = *K; Kt.ne[1] = this_kv; Kt.data = (char *)K->data + (size_t)kv0*K->nb[1]; +@@ -1784,10 +1799,11 @@ + dst_t.data = O_tiles.ptr + (size_t)t * nel; + if (decode_lse) opencoti_fattn_dst_lse = lse_t; + ggml_cuda_flash_attn_ext(ctx, &dst_t); ++ lse_emitted = decode_lse && opencoti_fattn_dst_lse_written; + } + + // lse_t (this_kv==0 → empty tile: kernel writes -inf, combine ignores it). +- if (!decode_lse) { ++ if (!lse_emitted) { + streaming_lse_kernel<<>>( + (const char *)Q->data, + (const char *)K->data + (size_t)kv0*K->nb[1], diff --git a/patches/0099-dca-gemma4-wiring.patch b/patches/0099-dca-gemma4-wiring.patch new file mode 100644 index 0000000000000000000000000000000000000000..9d20a1d5b0cd75f541c4f14a5a048fd7711bbbc7 --- /dev/null +++ b/patches/0099-dca-gemma4-wiring.patch @@ -0,0 +1,140 @@ +opencoti patch 0099 — DCA: wire Gemma-4 builders (bug-2118, task #626 unblock) + +`--dca on` was a SILENT NO-OP on all Gemma-4 archs: 0078-dca.patch wired only +the Qwen builders; dca.cpp's iswa build_attn_dca overload sat uncalled by +gemma4.cpp / gemma4-assistant.cpp. Consequence: every historical Gemma x DCA +eval measured DCA-off (contamination catalog: docs/evaluations bug-2118 notes). + +Wiring: build_attn_inp_dca on the iswa base cache; per-layer dca_l gates +GLOBAL (non-SWA) layers only; Q/K rope moves inside build_attn_dca via the +dca_rope bundle (chunked-regime rope must happen per-band); the assistant's +Q-only shared-KV path passes (Qcur, nullptr, nullptr). SWA layers and DCA-off +are byte-identical (else-branch keeps the original build_attn call). + +Gate (bs2, A4B + assistant, 32k, 24k prompt): INTRA regime (chunk==ctx) text +byte-identical to DCA-off with identical accept (f16 0.677 both; q8q4 text +identical); cross-chunk f16 accept 0.939, needle PRESENT. + +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -1,4 +1,6 @@ + #include "models.h" ++#include "dca.h" // opencoti F5 dca bug-2118 ++#include "llama-kv-cache-iswa.h" // bug-2118: get_base() on the iswa mctx + + void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; +@@ -161,6 +163,14 @@ + // TODO: is causal == true correct? might need some changes + auto * inp_attn = build_attn_inp_kv_iswa(); + ++ // opencoti F5 dca bug-2118 — Gemma-4 DCA wiring. 0078 wired only the Qwen builders; ++ // the iswa build_attn_dca overload (dca.cpp) existed uncalled, so '--dca on' was a ++ // silent no-op on Gemma. DCA applies to the GLOBAL (non-SWA) layers only (the SWA ++ // window is <= chunk already); the overload stores PRE-rope K + handles has_kv==false ++ // shared-KV globals (null k/v -> read the earlier layer's cache). ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx->get_base()) : nullptr; ++ + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // opencoti-hook: gemma4-mtp-hidden (bug-858 dual-context MTP) — when a context extracts the +@@ -202,6 +212,10 @@ + freq_factors = model.layers[il].rope_freqs; + } + ++ // opencoti F5 dca bug-2118 — global layers route through build_attn_dca (which ++ // ropes Q per regime and pre-ropes K at insert), so skip the standard pre-rope. ++ const bool dca_l = use_dca && !hparams.is_swa(il); ++ + // Q projection (shared for both non-KV and KV layers) + // this is to mirror Gemma4Attention in pytorch code + ggml_tensor * Qcur; +@@ -214,9 +228,11 @@ + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + ++ if (!dca_l) { // bug-2118: DCA ropes Q per regime in build_attn_dca + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_pos", il); ++ } + } + + // self-attention +@@ -238,14 +254,28 @@ + cb(Kcur, "Kcur_normed", il); + cb(Vcur, "Vcur_normed", il); + ++ if (!dca_l) { // bug-2118: DCA pre-ropes K at insert in build_attn_dca + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Kcur, "Kcur_pos", il); ++ } + ++ if (dca_l) { // bug-2118 (wo_s dropped like the qwen3 DCA path — null on these GGUFs) ++ const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr, ++ Qcur, Kcur, Vcur, rp, hparams.f_attention_scale, il); ++ } else { + cur = build_attn(inp_attn, model.layers[il].wo, + nullptr, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, + hparams.f_attention_scale, il); ++ } ++ } else if (dca_l) { ++ // bug-2118: shared-KV global layer under DCA — Q-only read of the earlier ++ // layer's DCA-formatted cache (null k/v -> no store). ++ const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr, ++ Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, il); + } else { + // reuse KV cache of earlier layers + cur = build_attn(inp_attn, +--- a/llama.cpp/src/models/gemma4-assistant.cpp ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -1,5 +1,7 @@ + // opencoti F5 M6-S4 mtp + #include "models.h" ++#include "dca.h" // opencoti F5 dca bug-2118 ++#include "llama-kv-cache-iswa.h" // bug-2118: get_base() on the iswa mctx + + #include + #include +@@ -190,6 +192,13 @@ + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + ++ // opencoti F5 dca bug-2118 — when the TARGET runs DCA, the drafter's global layers ++ // read the target's DCA-formatted (pre-roped, position-remapped) shared K/V, so its ++ // Q must be roped per regime through the same build_attn_dca path. Without this the ++ // drafter reads remapped keys with raw-extrapolated Q beyond native ctx -> accept 0. ++ const bool use_dca = cparams.dca_enabled; ++ llm_graph_input_dca * inp_dca = use_dca ? build_attn_inp_dca(inp_attn->mctx->get_base()) : nullptr; ++ + ggml_tensor * inpL = cur; + + for (int il = 0; il < n_layer; ++il) { +@@ -212,13 +221,22 @@ + cb(Qcur, "Qcur_normed", il); + + ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs; ++ const bool dca_l = use_dca && !is_swa; // bug-2118 ++ if (!dca_l) { + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, + freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_pos", il); ++ } + + // KV-less cross-attention into the shared/aliased target K/V (null k_cur/v_cur -> no store). ++ if (dca_l) { // bug-2118: Q-only per-regime read of the target's DCA cache ++ const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors }; ++ cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr, ++ Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, il); ++ } else { + cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr, + Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); ++ } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); diff --git a/patches/0100-dca-single-chunk-defer.patch b/patches/0100-dca-single-chunk-defer.patch new file mode 100644 index 0000000000000000000000000000000000000000..881bec5d4d3d05cd6946a5ec136e61bb0e2a9b4b --- /dev/null +++ b/patches/0100-dca-single-chunk-defer.patch @@ -0,0 +1,36 @@ +opencoti patch 0100 — DCA: defer quant lift in the single-chunk regime (bug-2119) + +dca.cpp gated defer_quant_lift on !single_chunk, so the single-chunk (INTRA) +regime whole-cache dca_lift_to_f16'd the quantized K/V EVERY forward — a +per-decode-step full-cache dequant that collapsed A4B 32k q8K/q4V decode to +4.69 tps. The comment assumed n_kv<=chunk meant "tiny cache"; at chunk 32768 +it is the entire context. Stock ggml_flash_attn_ext converts scalar-quant +natively, so the single-chunk branch defers exactly like analytical-bands. + +Gate (bs2, A4B q8K/q4V 32k chunk-32768): decode 4.69 -> 73.74 tps +(DCA-off 77.13); INTRA text remains byte-identical to DCA-off. + +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -255,16 +255,16 @@ + // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see + // ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case), so pass the quantized cache THROUGH instead of + // whole-cache materializing it to f16 via dca_lift_to_f16 every forward (the prefill cliff). Gate: +- // - !single_chunk: only the fused op carries the in-launch convert; single_chunk uses stock +- // ggml_flash_attn_ext (launch_fattn already converts scalar-quant natively) and runs only while +- // n_kv <= chunk (a tiny cache) so its lift cost is negligible — keep it on the safe lift path. +- // - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read. ++ // - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read. bug-2119: the ++ // single_chunk branch defers too — it uses stock ggml_flash_attn_ext, whose launch_fattn ++ // converts scalar-quant natively (the ordinary DCA-off decode path). The old !single_chunk gate ++ // assumed n_kv <= chunk means "tiny cache, lift is negligible", but chunk can be 32k+: the ++ // whole-cache two-leg graph lift ran EVERY decode step (A4B 32k/chunk-32768 q8q4: 76 -> 4.7 tps). + // - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the + // permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift. + // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output. + const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2]; + const bool defer_quant_lift = +- !inp_dca->single_chunk && + dca_is_inkernel_liftable(k_raw->type) && + dca_is_inkernel_liftable(v_raw->type) && + !v_trans_raw; diff --git a/patches/0101-kv-attn-rot-inherit.patch b/patches/0101-kv-attn-rot-inherit.patch new file mode 100644 index 0000000000000000000000000000000000000000..c13b55d7fa826bc34a935c90002d1d605b758bde --- /dev/null +++ b/patches/0101-kv-attn-rot-inherit.patch @@ -0,0 +1,49 @@ +opencoti patch 0101 — shared draft cache inherits attention-rotation state (bug-2120) + +llamafile 0.10.3 attention rotation (Hadamard-rotate K/V rows before quantized +store when the cache type is quantized && head_dim%64==0; readers rotate Q and +un-rotate the output via attn_inp_k_rot/v_rot) is computed per-cache from its +OWN type_k/type_v and accumulated head dims. The dual-context assistant draft +cache is created with f16 types AND its shared layers skip the head-dim +accumulation, so rotation silently disabled: the drafter read the target's +ROTATED quant K/V with an unrotated Q -> decorrelated attention, flat finite +logits, accept ~0.000 with ANY quant target KV (f16/f16 unaffected; Qwen NextN +unaffected — it reads through the target's own cache object). + +Fix: in the ctor share branch, inherit attn_rot_k/v, n_embd_head_*_all and the +Hadamard tables from the source cache, overriding the local computation. The +generic iswa build_attn then applies the rotation on the drafter's Q-only path. + +Gate (bs2, A4B + assistant Q8_0 drafter, 32k, 24k prompt): accept +k8_v16 0.000 -> 0.641; k8_v4 (production PolyKV) 0.000 -> 0.729; f16 control +0.677 unchanged. + +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -1106,6 +1106,26 @@ + } + } + ++ // opencoti bug-2120: a shared (dual-context drafter) cache must READ with the same ++ // attention-rotation state the SOURCE cache used to WRITE. The local computation above ++ // sees this cache's own type_k/type_v (f16 for the drafter) and n_embd_head_*_all == 0 ++ // (shared layers `continue` before accumulating), so it always disables rotation — the ++ // drafter then reads Hadamard-rotated quant K/V with an unrotated Q and un-unrotated ++ // output: decorrelated attention, accept ~0 with ANY quant target KV (f16/f16 unaffected ++ // because no rotation is applied at all). Inherit flags + head-dim extents + Hadamard ++ // tables from `other` so build_input_k_rot/v_rot emit the matching rotation inputs. ++ if (share && other) { ++ attn_rot_k = other->attn_rot_k; ++ attn_rot_v = other->attn_rot_v; ++ n_embd_head_k_all = other->n_embd_head_k_all; ++ n_embd_head_v_all = other->n_embd_head_v_all; ++ attn_rot_hadamard = other->attn_rot_hadamard; ++ if (attn_rot_k || attn_rot_v) { ++ LLAMA_LOG_INFO("%s: shared cache inherits attn_rot_k=%d attn_rot_v=%d from source (bug-2120)\n", ++ __func__, (int) attn_rot_k, (int) attn_rot_v); ++ } ++ } ++ + const char * LLAMA_KV_CACHE_DEBUG = getenv("LLAMA_KV_CACHE_DEBUG"); + debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0; + } diff --git a/patches/0102-fattn-bulk-h2d-scratch-ring.patch b/patches/0102-fattn-bulk-h2d-scratch-ring.patch new file mode 100644 index 0000000000000000000000000000000000000000..74eb683de0ce92a7a6c8a3fde899cd095f5ba0a7 --- /dev/null +++ b/patches/0102-fattn-bulk-h2d-scratch-ring.patch @@ -0,0 +1,154 @@ +opencoti patch 0102 — streaming-FA bulk-H2D tail lift + persistent staging scratch ring (bug-2116, bug-2121) + +Two changes to the rolling-KV streaming-FA lift in fattn.cu: + +1. bug-2116 (perf): for a HOST quant tail source at decode (n_b==1), bulk-copy + the RAW QUANT bytes H2D once at full DMA rate into device scratch, then run + the nc-converter from device memory. The converter reading the pinned-host + tail via UVA at word granularity measured 10-16 GB/s vs 57-65 GB/s bulk on + bs2 — the UVA cliff behind the S0 spill-decode curve. Measured after-curve + (14B-1M q8q4 window 131k, decode tps): 800MiB 9.05->12.23, 1600 3.96->10.28, + 6400 ~1->5.53. + +2. bug-2121 (correctness): both bulk paths (the new quant one and the #586 f16 + one) staged through ggml_cuda_pool_alloc scratch FREED on scope exit while + the async memcpy+convert on the copy stream were still pending. Decode + (bug-1841 single stream) is stream-ordered and safe; PREFILL runs two + streams and the FA compute allocates from the SAME pool concurrently (MMA + stream-k fixup buffers, every tile) -> freed-but-pending scratch re-handed + cross-stream -> corrupted K/V tiles. Manifested with >=2 host-tail tiles + (>32768 exercised tail cells): garbage decode / needle-LOST + racy host + GPFs; <=1 tail tile clean. Fix: persistent per-slot scratch ring + (g_scrK/g_scrV[4], grow-only, worst-case tile_kv_full capacity so no + cudaMalloc inside captured decode graphs) — no pool sharing across streams. + +Gate (bs2, 14B-1M q8q4 window 131k): tails 1600/3200/6400 all needle=PRESENT, +coherent text, zero crashes across three full 106k prefills (3200 previously +crashed 2x + garbage decode; 6400 previously needle-LOST). + +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -1350,6 +1350,30 @@ + char * const slotK_base = g_slotK; + char * const slotV_base = g_slotV; + ++ // opencoti-hook: f5-rolling-kv bug-2121 — PERSISTENT per-slot H2D staging scratch. ++ // The bulk-H2D lifts below (bug-2116 quant path + #586 f16 path) used ++ // ggml_cuda_pool_alloc scratch FREED on lambda/loop exit while the ++ // cudaMemcpyAsync + converter on the copy stream were still pending. On decode ++ // (single stream, bug-1841) reuse is stream-ordered and safe; on PREFILL the FA ++ // compute on ctx.stream() allocates from the SAME pool concurrently (MMA ++ // stream-k fixup buffers, every tile), so a freed-but-pending scratch gets ++ // re-handed to the other stream → cross-stream aliasing → corrupted K/V tiles. ++ // Manifested only with ≥2 host-tail tiles interleaving with compute (>32768 ++ // exercised tail cells): 14B-1M q8q4 window-mode tail=3200/6400 MiB → garbage ++ // decode / needle-LOST (+ racy host GPFs); tail≤1600 (1 tail tile) clean. A ++ // persistent ring keyed by slot removes pool sharing entirely; within-ring WAR ++ // is stream-ordered (memcpy → convert → next lift's memcpy, all on cs). ++ static char * g_scrK[4] = {}; static size_t g_scrK_cap[4] = {}; ++ static char * g_scrV[4] = {}; static size_t g_scrV_cap[4] = {}; ++ auto scr_get = [](char ** ring, size_t * cap, int slot, size_t need) -> char * { ++ if (cap[slot] < need) { ++ if (ring[slot]) CUDA_CHECK(cudaFree(ring[slot])); // cudaFree syncs pending users ++ CUDA_CHECK(cudaMalloc((void **)&ring[slot], need)); ++ cap[slot] = need; ++ } ++ return ring[slot]; ++ }; ++ + // R2-b S3b (#312): with >=2 slots the op runs a double-buffer ping-pong — the + // per-tile lift is issued on a dedicated copy_stream while FA/lse for the + // current tile run on ctx.stream(), so tile t+1's DMA overlaps tile t's +@@ -1445,10 +1469,34 @@ + // below are unchanged. Unifies device + pinned-host (S3c) sources + // (the kernel reads host via UVA). Slot is f16 → lse half-cast OK. + // 3c-5: tK/tV + loc select the window (device) or tail (host) source. +- to_fp16_k((const char *)tK->data + (size_t)loc*tK->nb[1], (half *)sK, ++ // opencoti-hook: f5-rolling-kv bug-2116 — for a HOST quant source, ++ // bulk-H2D the RAW QUANT bytes (cell-major block, 2.5× fewer bytes ++ // than f16) into device scratch at full DMA rate, then convert from ++ // device memory. Calling the converter on the host pointer made it ++ // read the tail via UVA at word granularity (~10–16 GB/s measured ++ // vs 57–65 GB/s bulk on bs2). n_b>1 keeps the UVA path (nb[3] ++ // batch blocks are not spanned by the cell-major copy; unused in ++ // the spill regime where ne[3]==1). ++ const bool bulkq = tr.host && n_b == 1; ++ const char * qsrcK = (const char *)tK->data + (size_t)loc*tK->nb[1]; ++ const char * qsrcV = (const char *)tV->data + (size_t)loc*tV->nb[1]; ++ if (bulkq) { ++ // bug-2121: persistent per-slot scratch — NOT pool memory (see scr_get above). ++ // Capacity = worst-case tile (tile_kv_full), not this_kv: this_kv grows per ++ // decode token, and a realloc inside a captured decode graph would fail. ++ const size_t kqblk = (size_t)this_kv * tK->nb[1]; ++ const size_t vqblk = (size_t)this_kv * tV->nb[1]; ++ char * qscrK = scr_get(g_scrK, g_scrK_cap, slot, (size_t)tile_kv_full * tK->nb[1]); ++ char * qscrV = scr_get(g_scrV, g_scrV_cap, slot, (size_t)tile_kv_full * tV->nb[1]); ++ CUDA_CHECK(cudaMemcpyAsync(qscrK, qsrcK, kqblk, cudaMemcpyDefault, cs)); ++ CUDA_CHECK(cudaMemcpyAsync(qscrV, qsrcV, vqblk, cudaMemcpyDefault, cs)); ++ qsrcK = qscrK; ++ qsrcV = qscrV; ++ } ++ to_fp16_k(qsrcK, (half *)sK, + head_dim, this_kv, n_head_kv, n_b, + (int64_t)(tK->nb[1]/ts_k), (int64_t)(tK->nb[2]/ts_k), (int64_t)(tK->nb[3]/ts_k), cs); +- to_fp16_v((const char *)tV->data + (size_t)loc*tV->nb[1], (half *)sV, ++ to_fp16_v(qsrcV, (half *)sV, + dv, this_kv, n_head_kv, n_b, + (int64_t)(tV->nb[1]/ts_v), (int64_t)(tV->nb[2]/ts_v), (int64_t)(tV->nb[3]/ts_v), cs); + } else +@@ -1468,17 +1516,20 @@ + const bool bulk = use_2d && tr.host; + const size_t kblk = (size_t)this_kv * tK->nb[1]; // contiguous cell-major span + const size_t vblk = (size_t)this_kv * tV->nb[1]; +- ggml_cuda_pool_alloc scrK(pool, bulk ? kblk : 1); +- ggml_cuda_pool_alloc scrV(pool, bulk ? vblk : 1); + const char * baseK; + const char * baseV; + if (bulk) { ++ // bug-2121: persistent per-slot scratch — NOT pool memory (see scr_get above). ++ // Sequential bb iterations reuse it WAR-safely (copy/repack both on cs). ++ // Worst-case tile capacity: see the quant path above (capture-safe growth). ++ char * scrK = scr_get(g_scrK, g_scrK_cap, slot, (size_t)tile_kv_full * tK->nb[1]); ++ char * scrV = scr_get(g_scrV, g_scrV_cap, slot, (size_t)tile_kv_full * tV->nb[1]); + const char * hK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1]; + const char * hV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1]; +- CUDA_CHECK(cudaMemcpyAsync(scrK.ptr, hK, kblk, cudaMemcpyDefault, cs)); // one full-BW H2D +- CUDA_CHECK(cudaMemcpyAsync(scrV.ptr, hV, vblk, cudaMemcpyDefault, cs)); +- baseK = scrK.ptr; // scratch is cell-major, cell `loc` at offset 0 +- baseV = scrV.ptr; ++ CUDA_CHECK(cudaMemcpyAsync(scrK, hK, kblk, cudaMemcpyDefault, cs)); // one full-BW H2D ++ CUDA_CHECK(cudaMemcpyAsync(scrV, hV, vblk, cudaMemcpyDefault, cs)); ++ baseK = scrK; // scratch is cell-major, cell `loc` at offset 0 ++ baseV = scrV; + } else { + baseK = (const char *)tK->data + (size_t)bb*tK->nb[3] + (size_t)loc*tK->nb[1]; + baseV = (const char *)tV->data + (size_t)bb*tV->nb[3] + (size_t)loc*tV->nb[1]; +@@ -1677,10 +1728,26 @@ + if (!all_f16) { + // S3d: one nc-converter launch per tile → packed f16 slot (see + // the overlap path for the layout proof). Source strides s=nb/ts. +- to_fp16_k((const char *)K->data + (size_t)kv0*K->nb[1], (half *)sK, ++ // bug-2116: host quant source → bulk-H2D the raw quant block at ++ // DMA rate + convert from device scratch; direct converter-on-host ++ // reads via UVA at word granularity (see the overlap-path note). ++ const bool bulkq = host_src && n_b == 1; ++ const size_t kqblk = bulkq ? (size_t)this_kv * K->nb[1] : 1; ++ const size_t vqblk = bulkq ? (size_t)this_kv * V->nb[1] : 1; ++ ggml_cuda_pool_alloc qscrK(pool, kqblk); ++ ggml_cuda_pool_alloc qscrV(pool, vqblk); ++ const char * qsrcK = (const char *)K->data + (size_t)kv0*K->nb[1]; ++ const char * qsrcV = (const char *)V->data + (size_t)kv0*V->nb[1]; ++ if (bulkq) { ++ CUDA_CHECK(cudaMemcpyAsync(qscrK.ptr, qsrcK, kqblk, cudaMemcpyDefault, stream)); ++ CUDA_CHECK(cudaMemcpyAsync(qscrV.ptr, qsrcV, vqblk, cudaMemcpyDefault, stream)); ++ qsrcK = qscrK.ptr; ++ qsrcV = qscrV.ptr; ++ } ++ to_fp16_k(qsrcK, (half *)sK, + head_dim, this_kv, n_head_kv, n_b, + (int64_t)(K->nb[1]/ts_k), (int64_t)(K->nb[2]/ts_k), (int64_t)(K->nb[3]/ts_k), stream); +- to_fp16_v((const char *)V->data + (size_t)kv0*V->nb[1], (half *)sV, ++ to_fp16_v(qsrcV, (half *)sV, + dv, this_kv, n_head_kv, n_b, + (int64_t)(V->nb[1]/ts_v), (int64_t)(V->nb[2]/ts_v), (int64_t)(V->nb[3]/ts_v), stream); + } else diff --git a/patches/0103-kv-cpu-fa-probe-guard.patch b/patches/0103-kv-cpu-fa-probe-guard.patch new file mode 100644 index 0000000000000000000000000000000000000000..b2b40b5fa73aacdbf20176e0a900219d1e4e184c --- /dev/null +++ b/patches/0103-kv-cpu-fa-probe-guard.patch @@ -0,0 +1,67 @@ +opencoti patch 0103 — rolling-KV CPU-FA cost-probe guard (bug-2125) + +The Stage-3d-S2 rolling-KV tactic selector runs a synthetic CPU flash-attn +"companion probe" (opencoti_fa_compute_probe_ms with on_cpu=true) purely to +MODEL a CPU_FA_TAIL cost for its audit log. CPU_FA_TAIL is gated out of +auto-select (#354) — the tail always streams on the GPU (POSITION_WINDOW) — so +the probe's result is never used for the decision. + +That probe builds a real ggml_flash_attn_ext over the actual cache K/V types and +runs it on the host CPU backend. For opencoti's CUDA-FA-VEC-only KV types (q6_0, +turbo{2,3,4,8}_0, turbo{2,3}_tcq) the CPU vec_dot trait is NULL, so the CPU +flash-attn dereferences a NULL fn-ptr → SIGSEGV at context-init (before any +client request). Stock types (f16/q8_0/q4_0/q5_0/q5_1/bf16) carry a CPU vec_dot +and survive, which is why only the custom-K tiers crashed under DCA at 256k/512k. + +Fix: gate the CPU probe on a host-CPU-FA-capability predicate +(opencoti_cpu_fa_capable_ktype) so it is never built for a K type the CPU +cannot process. Returning 0.0f just leaves CPU_FA_TAIL unmodeled; the selector +keeps its GPU-stream default. The rolling window stays a pure-GPU path for every +KV tier — no silent CPU fallback. Host-only change (no CUDA DSO impact). +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2795,6 +2795,28 @@ + // build_attn_mha FA contract (post-permute q[hd,n_q,n_head], k/v[hd,n_kv,n_head_kv], + // F16 mask). Never aborts boot: every shortfall (no GPU, unsupported op, non-GQA + // geometry, alloc miss) returns 0.0f and the selector keeps the stream default. ++// ++// opencoti bug-2125 SAFEGUARD: can the HOST CPU flash-attn path ++// (ggml_compute_forward_flash_attn_ext_f16) process a cache of this K type? It reads K via the ++// K-type's CPU vec_dot trait, which is NULL for opencoti's CUDA-FA-VEC-only KV types (q6_0, ++// turbo{2,3,4,8}_0, turbo{2,3}_tcq — GPU in-register readers, no CPU vec_dot / dp kernel). Running ++// CPU flash-attn over such a K deref's a NULL fn-ptr → SIGSEGV. The rolling-KV tail is ALWAYS ++// streamed on the GPU (POSITION_WINDOW; CPU_FA_TAIL is gated out of auto-select, #354), so the ONLY ++// place a host CPU flash-attn is ever built is the Stage-3d-S2 cost-model probe below — and its ++// result is audit-log only. Gate the CPU probe on this predicate so the rolling window stays a ++// pure-GPU path with NO silent CPU fallback for ANY KV tier. Stock ggml types (which DO carry a CPU ++// vec_dot) are the allow-list; every opencoti custom KV type is excluded. ++static bool opencoti_cpu_fa_capable_ktype(enum ggml_type t) { ++ switch (t) { ++ case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: ++ case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: ++ case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: ++ case GGML_TYPE_Q8_0: ++ return true; // stock ggml types: have a host CPU vec_dot ++ default: ++ return false; // opencoti custom KV (q6_0 / turbo* / turbo*_tcq): CUDA-only, NULL CPU vec_dot ++ } ++} + static float opencoti_fa_compute_probe_ms( + const llama_model & model, const llama_hparams & hparams, + int32_t il, ggml_type type_k, ggml_type type_v, int64_t bench_cells, +@@ -2823,6 +2845,15 @@ + return 0.0f; // device-type mismatch; selector falls back (stream default / no CPU model) + } + ++ // opencoti bug-2125 SAFEGUARD: never build a host CPU flash-attn over a K type the CPU cannot ++ // process (NULL vec_dot → SIGSEGV). This is audit-only (the tail always streams on GPU), so ++ // returning 0.0f just leaves CPU_FA_TAIL unmodeled — the selector keeps the GPU-stream default. ++ // Keeps the rolling window a pure-GPU path for q6_0 / turbo* / turbo*_tcq (and any future ++ // CUDA-only KV tier), with no silent CPU fallback. ++ if (on_cpu && !opencoti_cpu_fa_capable_ktype(type_k)) { ++ return 0.0f; ++ } ++ + const int64_t hd_k = (int64_t) hparams.n_embd_head_k(il); + const int64_t hd_v = (int64_t) hparams.n_embd_head_v(il); + const int64_t n_head = (int64_t) hparams.n_head(il); diff --git a/patches/0104-fattn-q6q4-vec-instance.patch b/patches/0104-fattn-q6q4-vec-instance.patch new file mode 100644 index 0000000000000000000000000000000000000000..9a0f0edfbb51237ada04c78ad5a9c91197a0fb60 --- /dev/null +++ b/patches/0104-fattn-q6q4-vec-instance.patch @@ -0,0 +1,71 @@ +opencoti patch 0104 — q6_0-K / q4_0-V FA-VEC instance (bug-2125 / #627) + +q6_0-K + q4_0-V (strong-K, cheap-V — the design invariant K precision >= V) is a +first-class asymmetric KV combo. Under GGML_CUDA_FA_ALL_QUANTS the kernel +selector admits K!=V and picks the VEC path, but no (q6_0, q4_0) FA-VEC instance +existed, so the D<=256 dispatch (Gemma-4 local/SWA layers — the global layers +use the DCA fused kernel's in-launch to_fp16_nc) fell through to +GGML_ABORT("fatal error") at ggml_cuda_flash_attn_ext_vec, crashing at boot. + +Fix (mirrors the q6_0-q5_0 pattern, #512/#81): add the instance TU +fattn-vec-instance-q6_0-q4_0.cu (head_dim 64/128/256), the +FATTN_VEC_CASES_ALL_D(Q6_0, Q4_0) dispatch case, and the build-functions.sh 3c +default-build inclusion (so the symbol is present without FA_ALL_QUANTS too). +Verified bs2 256k DCA-on: boots HEALTHY (was abort) + niah_single_1 = 100.0. +CUDA DSO change (needs a cuda rebuild). Only the q6_0/q4_0 direction is added — +reverse q4_0/q6_0 is intentionally NOT added (K must stay the higher-bit type). +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -353,13 +353,17 @@ + // ik_llama.cpp ships for q6_0 KV), registered OUTSIDE the FA_ALL_QUANTS block so they compile in + // the DEFAULT build — the matching instance .cu files are added unconditionally in + // collect_gpu_sources (build-functions.sh section 3c), like the turbo K==V instances below. +- // q6_0-q6_0 is the symmetric (default-reachable) cell; the 4 mixed cells are reachable only under ++ // q6_0-q6_0 is the symmetric (default-reachable) cell; the 5 mixed cells are reachable only under + // FA_ALL_QUANTS (the K!=V early-out in ggml_cuda_get_best_fattn_kernel), same as q5_0's mixed pairs. + FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q6_0) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q6_0) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q5_0) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_F16) + FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q6_0) ++ // opencoti-hook: q6_0 KV (#627/bug-2125) — strong-K q6_0 + cheap-V q4_0 asymmetric cell. Under ++ // FA_ALL_QUANTS the selector admits q6_0-K/q4_0-V, so the stock FA-VEC dispatch (local SWA layers) ++ // needs this instance; without it the D<=256 dispatch fell through to the GGML_ABORT below. ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_Q4_0) + + // opencoti-hook: turboquant perf-lift (Level B) — F5 M6-S2 #392. Fused turbo K/V vec path + // (always registered, independent of GGML_CUDA_FA_ALL_QUANTS). turbo2/3/4 at head_dim ∈ +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-q4_0.cu +@@ -0,0 +1,11 @@ ++// opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port) — strong-K + cheap-V asymmetric cell (#627/bug-2125). ++// q6_0-K / q4_0-V is a first-class KV combo (6-bit K precision, 4-bit V footprint); the local SWA layers ++// read it through the stock FA-VEC path (the global layers use the DCA fused kernel's to_fp16_nc dequant), ++// so this instance must exist or ggml_cuda_flash_attn_ext_vec_case is missing and the D<=256 ++// dispatch aborts at fattn.cu (GGML_ABORT "fatal error"). Mirrors fattn-vec-instance-q6_0-q5_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -162,7 +162,8 @@ + fi + + # 3c. opencoti-hook: q6_0 KV (Anbeeld/beellama.cpp port). q6_0 is a scalar KV type read by the +- # FA-VEC path; its 5 Anbeeld instances must compile in the DEFAULT build (same #468/bug-339 ++ # FA-VEC path; its 6 instances (5 Anbeeld + q6_0/q4_0 strong-K/cheap-V, #627) must compile in ++ # the DEFAULT build (same #468/bug-339 + # reasoning as the turbo block above — the default build does NOT glob fattn-vec, so without + # this the DSO lacks ggml_cuda_flash_attn_ext_vec_case and dlopen fails the instant + # q6_0 KV reaches flash-attention). No-op under FA_ALL_QUANTS=1 (already globbed) and on a +@@ -171,6 +172,7 @@ + for f in "$ti_dir"/fattn-vec-instance-q6_0-q6_0.cu \ + "$ti_dir"/fattn-vec-instance-q8_0-q6_0.cu \ + "$ti_dir"/fattn-vec-instance-q6_0-q5_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-q4_0.cu \ + "$ti_dir"/fattn-vec-instance-q6_0-f16.cu \ + "$ti_dir"/fattn-vec-instance-f16-q6_0.cu; do + [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" diff --git a/patches/0105-ws1-dca-per-tensor-defer.patch b/patches/0105-ws1-dca-per-tensor-defer.patch new file mode 100644 index 0000000000000000000000000000000000000000..e32dd22936b39ce5fd09a81d25d8156358fabf0b --- /dev/null +++ b/patches/0105-ws1-dca-per-tensor-defer.patch @@ -0,0 +1,71 @@ +opencoti patch 0105 — WS1 DCA per-tensor defer-decouple (bug-2127 / #628) + +The DCA (Dual-Chunk-Attention) single-chunk defer-lift gate (patch 0100) was +all-or-nothing across the K/V pair: it deferred the chunked-attention lift for +BOTH operands together, or neither. Under WS1's asymmetric scalar-K (q8_0) + +turbo/TCQ-V decode, the whole-K-cache path still ran the per-decode-step +cpy_q_f32 dequant of the entire K cache every step (~44% of GPU time) because +the V-side turbo defer forced the shared gate off for K too. + +Fix (host-only, dca.cpp — llm_graph_context::build_attn_dca_core): decouple the +defer decision per tensor (defer_k / defer_v) instead of one shared flag, so +scalar-K keeps its cheap in-place path while turbo-V defers independently. +Measured ~2× decode (bug-2127; the shipped WS1 throughput win alongside 0106). +Off-path (symmetric / non-DCA) is byte-identical. Host-only — no CUDA DSO +change. + +// opencoti-hook: WS1 asym scalar-K/turbo-V (#628, bug-2127) +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +index 01fb55d..ba747bf 100644 +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -263,11 +263,23 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the + // permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift. + // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output. ++ // opencoti-hook: WS1 asym scalar-K/turbo-V (#628, bug-2127) — the defer gate was ALL-OR-NOTHING ++ // across K and V (defer = liftable(K) && liftable(V) && !v_trans). An asymmetric scalar-K + ++ // turbo/TCQ-V config (e.g. -ctk q8_0 -ctv turbo2, the WS1 native pair) has liftable(K)=true but ++ // liftable(V)=false (turbo is not a to_fp16_nc scalar type), so the whole gate went false and ++ // dragged the q8_0 K OFF the fast in-launch path back onto the slow two-stage whole-cache graph ++ // lift (dca_lift_to_f16 → cpy_q_f32 dequant over the FULL 32k global-K cache EVERY decode step = ++ // ~44% of GPU time, the same bug-2119 cliff class it was meant to avoid). Decouple PER-TENSOR: ++ // scalar-quant K stays quantized → the fused launch's to_fp16_nc dequants it in-register; turbo/TCQ ++ // V keeps its existing fast single-pass dca_lift_to_f16 (a DIRECT ->f16 cast, not the two-stage). ++ // The fused launch (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case) and the single_chunk stock ++ // launch_fattn both convert K and V INDEPENDENTLY (fattn-mma-f16.cuh:2430-2436, K converted only ++ // if K->type != F16), so a mixed quant-K + f16-V input is fully supported and BYTE-IDENTICAL to the ++ // old both-lifted output (to_fp16_nc and dca_lift_to_f16 both compute d*q in f32 then round to f16). ++ // Pure perf change, gateable by logit-equivalence. + const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2]; +- const bool defer_quant_lift = +- dca_is_inkernel_liftable(k_raw->type) && +- dca_is_inkernel_liftable(v_raw->type) && +- !v_trans_raw; ++ const bool defer_k = dca_is_inkernel_liftable(k_raw->type); ++ const bool defer_v = dca_is_inkernel_liftable(v_raw->type) && !v_trans_raw; + + // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. + // opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16 +@@ -276,13 +288,15 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // post-permute cast below (unchanged); f16 V is untouched (byte-identical). + ggml_tensor * k_dca; + ggml_tensor * vp; +- if (defer_quant_lift) { +- // pass scalar-quant K/V straight to the fused op; permute(0,2,1,3) is a clean view (dim-0 = the +- // block axis, untouched) that the launch's to_fp16_nc dequants with the permuted strides. +- k_dca = k_raw; +- vp = ggml_permute(ctx0, v_raw, 0, 2, 1, 3); ++ ++ // K: defer scalar-quant to the launch (permute(0,2,1,3) is a clean dim-0/block-axis view the ++ // to_fp16_nc dequants with permuted strides); turbo/TCQ/bf16 → single-pass lift to f16. ++ k_dca = defer_k ? k_raw : dca_lift_to_f16(ctx0, k_raw); ++ ++ // V: defer scalar-quant straight to the fused op; otherwise lift-while-contiguous then permute. ++ if (defer_v) { ++ vp = ggml_permute(ctx0, v_raw, 0, 2, 1, 3); + } else { +- k_dca = dca_lift_to_f16(ctx0, k_raw); + ggml_tensor * v = v_raw; + if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { + v = dca_lift_to_f16(ctx0, v); diff --git a/patches/0106-ws1-scalar-k-turbo-v-boot-correctness.patch b/patches/0106-ws1-scalar-k-turbo-v-boot-correctness.patch new file mode 100644 index 0000000000000000000000000000000000000000..8255ec689aaae6e0a4c47500e52f98f39b54b0fd --- /dev/null +++ b/patches/0106-ws1-scalar-k-turbo-v-boot-correctness.patch @@ -0,0 +1,82 @@ +opencoti patch 0106 — WS1 scalar-K/turbo-V boot correctness (bug-2126 / #628) + +The rolling-KV boot microbench opencoti_fa_compute_probe_ms (llama-kv-cache.cpp) +builds a RAW ggml_flash_attn_ext directly over the cache K/V types (no +materializing cast) to time the compute tail. WS1 is the first config to put a +turbo/TCQ type on V while K is a stock scalar (q8_0/q6_0/q5_0/q4_0-K × +turbo{2,3}_{0,tcq}-V), which exposed two boot-time SIGSEGVs (null fn-ptr call, +rip=0) before any request: + + (1) CPU-FA probe: the type_k-only capability check let an asymmetric config + with a CPU-incapable V through — the CPU flash-attn reads BOTH operands + and deref'd a NULL to_float on the V read. Broadened to skip when type_k + OR type_v is not opencoti_cpu_fa_capable_ktype. + + (2) GPU probe: a D>256 asymmetric TCQ pair (Gemma-4 D512 GLOBAL layer, scalar + K + turbo*_tcq V) routes to the D512 MMA kernel, which converts the TCQ + operand via ggml_get_to_fp16_cuda() — NULL for TURBO*_TCQ (no TCQ convert + case) → call through 0x0. Added a GPU-side safeguard that returns 0.0f + (leaves the tail unmodeled; selector keeps the GPU-stream default) for the + asymmetric-TCQ, hd_k>256 case only. D≤256 asym-TCQ is VEC-safe (routes + in-register, no NULL to_fp16 path) after the 0107 fattn.cu selector, so it + is still probed. + +Both guards are audit-only (the rolling tail always streams on GPU), so they +change no served behavior — they only stop the boot crash. Host-only — no CUDA +DSO change. + +// opencoti-hook: WS1 scalar-K/turbo-V boot correctness (#628, bug-2126) +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 368ed2f..7713e70 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2845,15 +2845,43 @@ static float opencoti_fa_compute_probe_ms( + return 0.0f; // device-type mismatch; selector falls back (stream default / no CPU model) + } + +- // opencoti bug-2125 SAFEGUARD: never build a host CPU flash-attn over a K type the CPU cannot +- // process (NULL vec_dot → SIGSEGV). This is audit-only (the tail always streams on GPU), so +- // returning 0.0f just leaves CPU_FA_TAIL unmodeled — the selector keeps the GPU-stream default. +- // Keeps the rolling window a pure-GPU path for q6_0 / turbo* / turbo*_tcq (and any future +- // CUDA-only KV tier), with no silent CPU fallback. +- if (on_cpu && !opencoti_cpu_fa_capable_ktype(type_k)) { ++ // opencoti bug-2125 SAFEGUARD: never build a host CPU flash-attn over a K OR V type the CPU ++ // cannot process (NULL vec_dot on K / NULL to_float on V → SIGSEGV). The CPU flash-attn reads ++ // BOTH operands, so an asymmetric config with a stock K but a CUDA-only V (e.g. WS1's scalar-K ++ // q8_0 + turbo/tcq-V) still deref's a NULL fn-ptr on the V read — the type_k-only check let it ++ // through. This is audit-only (the tail always streams on GPU), so returning 0.0f just leaves ++ // CPU_FA_TAIL unmodeled — the selector keeps the GPU-stream default. Keeps the rolling window a ++ // pure-GPU path for q6_0 / turbo* / turbo*_tcq on EITHER axis, with no silent CPU fallback. ++ if (on_cpu && (!opencoti_cpu_fa_capable_ktype(type_k) || !opencoti_cpu_fa_capable_ktype(type_v))) { + return 0.0f; + } + ++ // opencoti bug-2126 GPU-side SAFEGUARD (WS1/#628): unlike the real decode path (build_attn_mha, ++ // which ggml_casts a turbo V → f16 IN THE GRAPH before FA), this probe builds a RAW ++ // ggml_flash_attn_ext directly over the cache types with NO materializing cast. For an asymmetric ++ // pair with a TCQ operand and no native D-specific FA-VEC instance — exactly Gemma-4's D512 GLOBAL ++ // layer with a scalar K (q8_0) + turbo*_tcq V — the op routes to the D512 MMA kernel, which needs ++ // f16 K AND V and converts the TCQ operand via ggml_get_to_fp16_cuda(). That returns NULL for ++ // TURBO*_TCQ (convert.cu has plain-turbo cases but no TCQ case → CUDA-FA-VEC-in-register only) → ++ // call through 0x0 → SIGSEGV at boot (before any request). The probe is audit-only (CPU_FA_TAIL is ++ // gated out of auto-select, #354; the tail always streams on GPU), so returning 0.0f just leaves it ++ // unmodeled — the selector keeps the GPU-stream default. Symmetric TCQ routes to the native turbo ++ // FA-VEC (in-register, no cast) and is NOT skipped; only an asymmetric TCQ pair is. ++ const auto opencoti_is_tcq_type = [](enum ggml_type t) { ++ return t == GGML_TYPE_TURBO2_TCQ || t == GGML_TYPE_TURBO3_TCQ; ++ }; ++ // opencoti WS1/#628: narrowed from all-asym-TCQ to hd_k>256 only. Post-WS1-fix the fattn.cu ++ // selector routes D128/256 asym turbo/TCQ-V to VEC (in-register, no NULL to_fp16 MMA path), so a ++ // fused raw-FA over D≤256 asym-TCQ is VEC-safe and can be probed. D512 asym-TCQ still routes to ++ // MMA→ggml_get_to_fp16_cuda()==NULL for TURBO*_TCQ → SIGSEGV, so keep skipping ONLY that. ++ { ++ const int64_t hd_k_guard = (int64_t) hparams.n_embd_head_k(il); ++ if (!on_cpu && type_k != type_v && ++ (opencoti_is_tcq_type(type_k) || opencoti_is_tcq_type(type_v)) && hd_k_guard > 256) { ++ return 0.0f; ++ } ++ } ++ + const int64_t hd_k = (int64_t) hparams.n_embd_head_k(il); + const int64_t hd_v = (int64_t) hparams.n_embd_head_v(il); + const int64_t n_head = (int64_t) hparams.n_head(il); diff --git a/patches/0107-ws1-turbo-mma-fused-decode.patch b/patches/0107-ws1-turbo-mma-fused-decode.patch new file mode 100644 index 0000000000000000000000000000000000000000..bbf21f47bcbcb7937a44ba01c0c1ce1cccd26ad2 --- /dev/null +++ b/patches/0107-ws1-turbo-mma-fused-decode.patch @@ -0,0 +1,1243 @@ +opencoti patch 0107 — WS1 turbo-MMA verify + scalar-K/turbo-V fused decode (#628/#630, bug-2130) + +The WS1 asymmetric scalar-K (q8_0/q6_0/q5_0/q4_0) + cheap turbo/TCQ-V decode +family, in two coordinated pieces, plus a turbo-aware tensor-core VERIFY kernel. + +1. FUSED-VEC DECODE (n_q=1) — the foundation. New FA-VEC template-instances for + strong scalar-K × turbo/TCQ-V (K read as scalar via vec_dot_KQ, V read + in-register as turbo/TCQ via dequantize_V — no f16 materialize), the + FATTN_VEC_CASES_ALL_D dispatch cases (fattn.cu), the K!=V selector admission + (scalar_k_turbo_v, like both_tcq), and the default-build inclusion + (build-functions.sh 3e — the graph emits scalar-K/turbo-V to FA at decode, so + the DSO must carry these or dlopen fails when that KV first reaches FA). The + graph (build_attn_mha wht_q/wht_o split) applies the OUTPUT inverse-WHT ONLY + (V is FWHT-rotated, K is true-space). Design invariant: K bit-width > V; + reverse (turbo-K/scalar-V) intentionally NOT built. + +2. TURBO-MMA VERIFY (n_q>1) — OPT-IN, default-OFF. Turbo-aware MMA kernels that + dequant q8_0-K + turbo2-V into the MMA smem tile (rotated V; graph wht_o keeps + it exact) for MTP verify batches, at head_dim 128 (Qwen full-attn), 256 + (Gemma SWA-local) and 512 (Gemma global). Three per-head-dim knobs + (WS1_TURBO_MMA_D128/D256/D512), all default-OFF: when off the verify batch + keeps the fused-VEC routing (the pre-#628 default + logit-equiv reference). + The fattn.cu selector routes verify (n_q>1, knob on) to MMA_F16; the + ggml_cuda_flash_attn_ext_mma_f16 dispatch hands q8_0/turbo2 to the turbo + instance; the llama-graph.cpp fuse path gates D512 verify behind the knob. + De-rotate (WS1_DEROTATE) is a separate default-OFF experiment (inverse-FWHT V + in-load so the graph skips wht_o); default-off ⇒ byte-identical. + + VERDICT (#630, 3-model crossover map A4B / omni27 / qw35a3, tps+TV+niah): + turbo-MMA is mid-ctx-specialized and earns NO default-on anywhere. On Gemma + A4B/D256 it wins ≤128k (+4–6%) but net-LOSES @256k (−14%, the windowed-D256 + cast-saving is swamped by the ctx-growing D512 global-cast tail) and is + un-gatable (SWA-windowed op n_kv). On Qwen D128 (dense omni27 / MoE qw35a3) + it is bit-exact (real_frac 0, niah 100) but tps-flat — the FFN dominates the + verify step, so attention acceleration has no leverage. Shipped OPT-IN, + default-OFF; the instances are additive/correct/harmless. + +bug-2130: the MMA _case passes !is_quant_kv for need_f16_K/need_f16_V so +launch_fattn does NOT pre-cast quant K/V to f16 (f16 keeps true,true → +byte-identical; turbo/q8_0 pass false,false to read raw blocks). + +CUDA DSO change (needs a cuda rebuild). + +// opencoti-hook: WS1 turbo-MMA (#628/#630) + scalar-K/turbo-V fused decode +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index 04a81e7..2eb1fc7 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -135,6 +135,28 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con + } + } + ++// opencoti-hook: WS1 turbo-MMA (#628) — scalar-K/turbo-V verify entries (template-instances/ ++// fattn-mma-f16-turbo-q8_0-turbo2{,-d512}.cu). Decode q8_0-K + turbo2-V into the MMA smem tile (rotated V). ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++ ++// opencoti-hook: WS1 turbo-MMA (#628/#630) — per-head-dim OPT-IN knobs (default OFF). D256 = Gemma SWA-local, ++// D512 = Gemma global, D128 = Qwen full-attention. When off, the scalar-K/turbo-V verify batch keeps the ++// fused-VEC routing (the pre-#628 default + logit-equiv reference). ++static bool ws1_turbo_mma_d256_enabled() { ++ static const bool en = []() { const char * e = getenv("WS1_TURBO_MMA_D256"); return (e && e[0] == '1'); }(); ++ return en; ++} ++static bool ws1_turbo_mma_d512_enabled() { ++ static const bool en = []() { const char * e = getenv("WS1_TURBO_MMA_D512"); return (e && e[0] == '1'); }(); ++ return en; ++} ++static bool ws1_turbo_mma_d128_enabled() { ++ static const bool en = []() { const char * e = getenv("WS1_TURBO_MMA_D128"); return (e && e[0] == '1'); }(); ++ return en; ++} ++ + static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const ggml_tensor * KQV = dst; +@@ -143,6 +165,15 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + ++ // opencoti-hook: WS1 turbo-MMA (#628): the selector routes scalar-K (q8_0) + turbo2-V verify batches ++ // (D256 local OR D512 global, n_q>1) to MMA_F16; hand them to the turbo-aware instance instead of the ++ // f16 Q->ne[0] switch (which would reinterpret the turbo/q8_0 blocks as raw f16). Others keep the switch. ++ if (K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_TURBO2_0 && Q->ne[0] == V->ne[0]) { ++ if (Q->ne[0] == 128) { ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d128(ctx, dst); return; } ++ if (Q->ne[0] == 256) { ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2(ctx, dst); return; } ++ if (Q->ne[0] == 512) { ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ctx, dst); return; } ++ } ++ + switch (Q->ne[0]) { + case 64: + GGML_ASSERT(V->ne[0] == 64); +@@ -393,6 +424,28 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_TCQ, GGML_TYPE_TURBO2_TCQ) + FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_TCQ, GGML_TYPE_TURBO3_TCQ) + ++ // opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC ++ // cells (K bit-width > V-bit-width; reverse NOT added). K read as scalar (vec_dot_KQ, Q q8_1), V read ++ // in-register as turbo/TCQ (dequantize_V) — no f16 materialize. The graph (build_attn_mha wht_q/wht_o ++ // split) applies the OUTPUT inverse-WHT ONLY (V is FWHT-rotated), NO forward-Q-WHT (K is true-space). ++ // Fused at head_dim {128,256} decode; D64 registered for completeness (materialize path at 64/512). ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_0) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_TCQ) ++ FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_TCQ) ++ + // opencoti #493 (F5 M6-S4): head_dim 512 f16 VEC dispatch — the runtime counterpart to the + // selector's D512/gqa<=2 f16 VEC route (ggml_cuda_get_best_fattn_kernel) + the f16-f16 D512 + // instance. Without this case the selector picks VEC but the dispatch falls through to the abort +@@ -509,7 +562,15 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + { + const bool both_tcq = (K->type == GGML_TYPE_TURBO3_TCQ || K->type == GGML_TYPE_TURBO2_TCQ) && + (V->type == GGML_TYPE_TURBO3_TCQ || V->type == GGML_TYPE_TURBO2_TCQ); +- if (K->type != V->type && !both_tcq) { ++ // opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric strong scalar-K (q4_0/q5_0/q6_0/ ++ // q8_0) + cheap turbo/TCQ-V has dedicated in-register FA-VEC instances (K read as scalar, V read as ++ // turbo/TCQ), so admit it like both_tcq; every other K!=V still falls back. K bit-width > V; no reverse. ++ const bool scalar_k_turbo_v = ++ (K->type == GGML_TYPE_Q4_0 || K->type == GGML_TYPE_Q5_0 || ++ K->type == GGML_TYPE_Q6_0 || K->type == GGML_TYPE_Q8_0) && ++ (V->type == GGML_TYPE_TURBO2_0 || V->type == GGML_TYPE_TURBO3_0 || ++ V->type == GGML_TYPE_TURBO2_TCQ || V->type == GGML_TYPE_TURBO3_TCQ); ++ if (K->type != V->type && !both_tcq && !scalar_k_turbo_v) { + return BEST_FATTN_KERNEL_NONE; + } + } +@@ -554,8 +615,31 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const + // VEC kernel (centroid×norm read in-register); never route them to MMA/WMMA/TILE, which can't + // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when + // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift. +- if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0 || +- K->type == GGML_TYPE_TURBO3_TCQ || K->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ FA-VEC decode (#447/#500) ++ // opencoti-hook: WS1 scalar-K/turbo-V (#628, bug-2126) — key on turbo/tcq on EITHER K OR V. The ++ // asymmetric scalar-K + turbo/TCQ-V family (q8_0/q6_0/q5_0/q4_0-K × turbo{2,3}_{0,tcq}-V) is the ++ // first to put turbo on V while K is scalar; a K-only guard let n_q>2 MTP-verify batches fall ++ // through to MMA_F16, which reinterprets the packed turbo/TCQ-V blocks as raw f16 → graded garbage ++ // (plain-turbo degrades, TCQ → NaN → empty completions). MMA/TILE cannot decode turbo V either. ++ const auto is_turbo_or_tcq_type = [](enum ggml_type t) { ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || ++ t == GGML_TYPE_TURBO3_TCQ || t == GGML_TYPE_TURBO2_TCQ; ++ }; ++ if (is_turbo_or_tcq_type(K->type) || is_turbo_or_tcq_type(V->type)) { // opencoti-hook: TCQ FA-VEC decode (#447/#500) ++ // opencoti-hook: WS1 turbo-MMA (#628, bug-2126) — the scalar-K (q8_0) + turbo2-V family now has a ++ // turbo-AWARE tensor-core VERIFY kernel (dequant into the MMA smem tile; graph wht_o keeps it exact). ++ // Route verify batches (D256, n_q>1) to MMA_F16; decode (n_q==1) and every other turbo/tcq pair keep ++ // the fused VEC path (its design point). The graph only emits turbo to FA for tbv_nq_per_strm<=8, so ++ // any turbo seen here at n_q>1 is a fused verify batch within the wht_o-wrapped window. ++ // D256 local (SWA) and D512 global both route verify (n_q>1) to the turbo-aware MMA; the D512 global ++ // layers are the long-ctx-dominant ones (#628 D512 generalization). Decode (n_q=1) keeps VEC/materialize. ++ if (turing_mma_available(cc) && Q->ne[1] > 1 && ++ K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_TURBO2_0 && Q->ne[0] == V->ne[0] && ++ ((Q->ne[0] == 128 && ws1_turbo_mma_d128_enabled()) || ++ (Q->ne[0] == 256 && ws1_turbo_mma_d256_enabled()) || ++ (Q->ne[0] == 512 && ws1_turbo_mma_d512_enabled())) && ++ K->ne[1] % FATTN_KQ_STRIDE == 0) { ++ return BEST_FATTN_KERNEL_MMA_F16; ++ } + // opencoti #411 (F5 M6-S2 Level-B): turbo2/3 also ship a D=512 fused VEC instance for Gemma-4 + // GLOBAL layers (key_length=512) so the 256k global cache reads in-register instead of the + // Level-A turbo→f16 materialize. turbo4/turbo8 have no D=512 instance → keep the ≤256 cap +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +index 185eaef..bd94bfb 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -2,6 +2,10 @@ + #include "cp-async.cuh" + #include "mma.cuh" + #include "fattn-common.cuh" ++// opencoti-hook: WS1 turbo-MMA (#628) — TURBO3_WHT_SIGNS1/2 + inv_sqrt128 for the de-rotate-in-load ++// fidelity variant (inverse-FWHT of turbo-V into TRUE-space f16). #pragma once guarded; tables are ++// width-independent (per turbo-dequant.cuh), reused by the materialize cpy_turbo*_f16 path. ++#include "turbo-dequant.cuh" + + using namespace ggml_cuda_mma; + +@@ -447,6 +451,114 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( + } + } + ++// opencoti-hook: WS1 turbo-MMA (#628) — dequant-load variant of flash_attn_ext_f16_load_tile. ++// Decodes quantized K/V blocks directly into the MMA smem tile (rotated-f16 for turbo, plain-f16 for ++// q8_0) via the shared dequantize_V_t readers, in place of the raw cp.async/memcpy copy. cp.async ++// cannot dequant, so the scalar-K/turbo-V MMA instances run nstages=0 (manual load only). K/V stay in ++// their STORED space (q8_0=true, turbo=FWHT-rotated); the graph wht_q/wht_o wrapping (build_attn_mha) ++// makes attention exact in true space. `KV` is the byte base of key-block row 0 for this k-tile; ++// `stride_KV_bytes` is the per-row byte stride (K->nb[1] / V->nb[1]); `elem0` is the element (D) offset ++// of the first loaded column. Mirrors the manual (non-cp.async) branch of flash_attn_ext_f16_load_tile. ++template ++static __device__ __forceinline__ void flash_attn_ext_f16_load_tile_dequant( ++ const char * const __restrict__ KV, half2 * const __restrict__ tile_KV, ++ const int D2, const int stride_KV_bytes, const int i_sup, const int elem0) { ++ constexpr int warp_size = ggml_cuda_get_physical_warp_size(); ++ constexpr int h2_per_chunk = 16/sizeof(half2); // 4 half2 == 8 elements per thread-chunk ++ constexpr int ne_per_chunk = 2*h2_per_chunk; // 8 ++ constexpr dequantize_V_t dequantize_V = get_dequantize_V(); ++ const int chunks_per_row = D2 / h2_per_chunk; ++ auto load = [&] __device__ (const int n) { ++ const int stride_k = 32 >> n; ++ const int k0_start = stride_k == 32 ? 0 : chunks_per_row - chunks_per_row % (2*stride_k); ++ const int k0_stop = chunks_per_row - chunks_per_row % (1*stride_k); ++ const int stride_i = warp_size / stride_k; ++ ++ if (k0_start == k0_stop) { ++ return; ++ } ++ ++#pragma unroll ++ for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*stride_i) { ++ const int i = i0 + threadIdx.y*stride_i + (stride_k == warp_size ? 0 : threadIdx.x / stride_k); ++ ++ if (i0 + nwarps*stride_i > nbatch_fa && i >= nbatch_fa) { ++ break; ++ } ++ ++#pragma unroll ++ for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { ++ const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); ++ ++ half2 * dst = tile_KV + i*stride_tile + k*h2_per_chunk; ++ if (!oob_check || i < i_sup) { ++ dequantize_V(KV + i*stride_KV_bytes, dst, elem0 + k*ne_per_chunk); ++ } else { ++#pragma unroll ++ for (int c = 0; c < h2_per_chunk; ++c) { ++ dst[c] = make_half2(0.0f, 0.0f); ++ } ++ } ++ } ++ } ++ }; ++ ggml_cuda_unroll<6>{}(load); ++} ++ ++// opencoti-hook: WS1 turbo-MMA (#628) — de-rotate knob. Set once (per turbo-TU copy) from the ++// WS1_DEROTATE env by the turbo dispatch; read in flash_attn_ext_f16_iter. 0 = keep rotated-V in the ++// smem tile + graph wht_o (default, byte-identical to the shipped rotated-MMA variant). 1 = inverse-FWHT ++// the rotated-V tile to TRUE space in-load; the graph MUST then skip wht_o (else double de-rotate). ++static __constant__ int d_ws1_derotate = 0; ++ ++// opencoti-hook: WS1 turbo-MMA (#628) — in-place f16 inverse-FWHT of the loaded (rotated) turbo-V smem ++// tile → TRUE-space f16, so the tensor cores accumulate P·V_true directly (like materialize) instead of ++// P·V_rot + graph wht_o. Mirrors turbo-cpy.cu turbo_wht_inv_write EXACTLY (signs2 → 128-pt butterfly → ++// 1/√128·signs1, scale_inv=nullptr — the fused path's wht_o also passes scale=nullptr). Math is identical ++// to rotated+wht_o (H commutes with the softmax·V contraction); only the f16-vs-f32 butterfly precision ++// differs. `n_elem` is the resident V-column span for this slice (== DV for the single-slice D256 config, ++// block-aligned: n_elem%128==0). All threads participate; no divergent early-return before the syncs. ++template ++static __device__ __forceinline__ void flash_attn_ext_f16_derotate_tile_V(half2 * const __restrict__ tile_V, const int n_elem) { ++ half * tv = (half *) tile_V; ++ const int row_h = 2*stride_tile_V; // half stride between V rows in the tile ++ const int nblk = n_elem >> 7; // 128-blocks per row (2 for D256) ++ const int nbr = nbatch_fa * nblk; // total (row, block) pairs ++ const int tid = threadIdx.y*ggml_cuda_get_physical_warp_size() + threadIdx.x; ++ // Phase 1: ×signs2 ++ for (int idx = tid; idx < nbr*128; idx += nthreads) { ++ const int br = idx >> 7, j = idx & 127; ++ const int row = br / nblk, blk = br - row*nblk; ++ half * base = tv + row*row_h + blk*128; ++ base[j] = __hmul(base[j], __float2half(TURBO3_WHT_SIGNS2[j])); ++ } ++ __syncthreads(); ++ // Phase 2: 128-point Walsh-Hadamard butterfly (7 stages) ++#pragma unroll ++ for (int h = 1; h < 128; h <<= 1) { ++ const int bb = 31 - __clz(h); ++ for (int p = tid; p < nbr*64; p += nthreads) { ++ const int br = p / 64, pp = p & 63; ++ const int row = br / nblk, blk = br - row*nblk; ++ const int lo = pp & (h - 1); ++ const int j = (pp >> bb << (bb + 1)) | lo; // insert 0 at bit bb ⇒ (j & h)==0 ++ half * base = tv + row*row_h + blk*128; ++ const half a = base[j], c = base[j | h]; ++ base[j] = __hadd(a, c); ++ base[j | h] = __hsub(a, c); ++ } ++ __syncthreads(); ++ } ++ // Phase 3: ×(1/√128)·signs1 ++ for (int idx = tid; idx < nbr*128; idx += nthreads) { ++ const int br = idx >> 7, j = idx & 127; ++ const int row = br / nblk, blk = br - row*nblk; ++ half * base = tv + row*row_h + blk*128; ++ base[j] = __hmul(base[j], __float2half(0.08838834764831845f * TURBO3_WHT_SIGNS1[j])); ++ } ++ __syncthreads(); ++} ++ + template + static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( + const half * const __restrict__ mask_h, half * const __restrict__ tile_mask, +@@ -531,7 +643,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( + template ++ bool dca_fused = false, ++ ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16> // opencoti-hook: WS1 turbo-MMA (#628) + static __device__ __forceinline__ void flash_attn_ext_f16_iter( + const float2 * const __restrict__ Q_f2, + const half2 * const __restrict__ K_h2, +@@ -580,7 +693,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + // below) -> tile_K read uninitialised -> NaN KQ -> garbage ('!' loop). DKQ=512 already gets nstages=1 + // (why Gemma's fused path works). Clamp the fused path to single-stage so 128/256 mirror the validated 512. + constexpr int nstages_cfg = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); +- constexpr int nstages = dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg; ++ // opencoti-hook: WS1 turbo-MMA (#628): quantized K/V decode in the manual load branch; cp.async ++ // cannot dequant, so force single-buffered manual loading (nstages=0 ⇒ use_cp_async=false). ++ constexpr bool is_quant_kv = (type_K != GGML_TYPE_F16) || (type_V != GGML_TYPE_F16); ++ constexpr int nstages = is_quant_kv ? 0 : (dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg); + + constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_K = nbatch_K2 + 4; +@@ -622,8 +738,14 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + + if constexpr (nstages <= 1) { + constexpr bool use_cp_async = nstages == 1; +- flash_attn_ext_f16_load_tile +- (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup); ++ if constexpr (type_K != GGML_TYPE_F16) { // opencoti-hook: WS1 turbo-MMA (#628) ++ // stride_K carries BYTES for quant K (see kernel entry); k0_start is a half2 column ⇒ 2*k0_start elements. ++ flash_attn_ext_f16_load_tile_dequant ++ ((const char *) K_h2 + int64_t(k_VKQ_0)*stride_K, tile_K, k0_diff, stride_K, k_VKQ_sup, 2*k0_start); ++ } else { ++ flash_attn_ext_f16_load_tile ++ (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup); ++ } + if (use_cp_async) { + cp_async_wait_all(); + } +@@ -975,14 +1097,30 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + if constexpr (nstages <= 1) { + if (!V_is_K_view || i0_stop > 2*nbatch_K2) { + constexpr bool use_cp_async = nstages == 1; +- flash_attn_ext_f16_load_tile +- (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup); ++ if constexpr (type_V != GGML_TYPE_F16) { // opencoti-hook: WS1 turbo-MMA (#628) ++ // stride_V carries BYTES for quant V; i0_start is a logical element (D) offset. ++ flash_attn_ext_f16_load_tile_dequant ++ ((const char *) V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, i0_diff/2, stride_V, k_VKQ_sup, i0_start); ++ } else { ++ flash_attn_ext_f16_load_tile ++ (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup); ++ } + if (use_cp_async) { + cp_async_wait_all(); + } + __syncthreads(); + } + } ++ // opencoti-hook: WS1 turbo-MMA (#628) — de-rotate the just-loaded rotated-V tile to TRUE space ++ // (in-place f16 inverse-FWHT) so the tensor cores accumulate P·V_true; the graph then skips wht_o. ++ // Runtime-flagged (d_ws1_derotate, set only for the turbo case) so the rotated variant stays the ++ // default/byte-identical A/B baseline. Single-slice D256 (nbatch_V2=128 ⇒ i0_diff=DV, block-aligned). ++ if constexpr (type_V != GGML_TYPE_F16) { ++ if (d_ws1_derotate) { ++ flash_attn_ext_f16_derotate_tile_V(tile_V, i0_diff); ++ __syncthreads(); ++ } ++ } + const half2 * tile_V_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? tile_V : tile_V + i0_start/2; + + #if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +@@ -1176,7 +1314,8 @@ static __device__ __forceinline__ void dca_load_tile_Q( + } + } + +-template ++template // opencoti-hook: WS1 turbo-MMA (#628) + static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + const float2 * const __restrict__ Q_f2, + const half2 * const __restrict__ K_h2, +@@ -1243,7 +1382,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + // rationale). nstages drives the tile_V/tile_mask shared-mem offsets just below (~:1126); the kernel, + // iter, and the host ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case smem sizing must all use this clamp. + constexpr int nstages_cfg = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); +- constexpr int nstages = dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg; ++ // opencoti-hook: WS1 turbo-MMA (#628): mirror iter's force-0 for quant K/V (manual dequant load, ++ // no cp.async). Governs the tile_V/tile_mask smem offsets below; must match the host _turbo_case sizing. ++ constexpr bool is_quant_kv = (type_K != GGML_TYPE_F16) || (type_V != GGML_TYPE_F16); ++ constexpr int nstages = is_quant_kv ? 0 : (dca_fused && nstages_cfg > 1 ? 1 : nstages_cfg); + + if (cols_per_warp > ncols) { + NO_DEVICE_CODE; +@@ -1502,7 +1644,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + constexpr int k_VKQ_sup = nbatch_fa; + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/false, type_K, type_V> // opencoti-hook: WS1 turbo-MMA (#628) + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, + KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); +@@ -1511,7 +1653,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + const int k_VKQ_sup = ne11 - kb0*nbatch_fa; + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/false, type_K, type_V> // opencoti-hook: WS1 turbo-MMA (#628) + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, + KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); +@@ -1522,7 +1664,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + constexpr int k_VKQ_sup = nbatch_fa; + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/false, type_K, type_V> // opencoti-hook: WS1 turbo-MMA (#628) + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, + KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); +@@ -1531,7 +1673,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + constexpr int k_VKQ_sup = nbatch_fa; + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/false, type_K, type_V> // opencoti-hook: WS1 turbo-MMA (#628) + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, + KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); +@@ -1926,7 +2068,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + #endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + } + +-template ++template // opencoti-hook: WS1 turbo-MMA (#628) + __launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2)) + static __global__ void flash_attn_ext_f16( + const char * __restrict__ Q, +@@ -2004,10 +2147,12 @@ static __global__ void flash_attn_ext_f16( + + const int stride_Q1 = nb01 / sizeof(float2); + const int stride_Q2 = nb02 / sizeof(float2); +- const int stride_K = nb11 / sizeof(half2); ++ // opencoti-hook: WS1 turbo-MMA (#628): for quantized K/V, keep the stride in BYTES (nb11/nb21) so the ++ // dequant load path addresses turbo/q8_0 block rows directly; f16 keeps the half2-count stride. ++ const int stride_K = (type_K == GGML_TYPE_F16) ? int(nb11 / sizeof(half2)) : int(nb11); + const int stride_mask = nb31 / sizeof(half); + +- const int stride_V = V_is_K_view ? stride_K : nb21 / sizeof(half2); ++ const int stride_V = V_is_K_view ? stride_K : ((type_V == GGML_TYPE_F16) ? int(nb21 / sizeof(half2)) : int(nb21)); + + const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa; + const int iter_j = (ne01.z + (ncols1 - 1)) / ncols1; +@@ -2054,12 +2199,12 @@ static __global__ void flash_attn_ext_f16( + constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. + if (kb0_start == 0) { + constexpr bool needs_fixup = false; // CUDA block is working on an entire tile. +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + } else { + constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile. +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + } +@@ -2102,7 +2247,7 @@ static __global__ void flash_attn_ext_f16( + + constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. + constexpr bool needs_fixup = false; +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop, kb0_min); + #else +@@ -2119,7 +2264,8 @@ static __global__ void flash_attn_ext_f16( + #endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)) + } + +-template ++template // opencoti-hook: WS1 turbo-MMA (#628) + void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * KQV = dst; + const int id = ggml_cuda_get_device(); +@@ -2133,7 +2279,10 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + const int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols, cc); + const int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols, cc); + const bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols, cc); +- const int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, cc); ++ // opencoti-hook: WS1 turbo-MMA (#628): quantized K/V run manual dequant load (nstages=0) — must match ++ // the kernel/process_tile/iter force-0 so the 1-stage smem sizing below is used. ++ const bool is_quant_kv = (type_K != GGML_TYPE_F16) || (type_V != GGML_TYPE_F16); ++ const int nstages = is_quant_kv ? 0 : ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, cc); + + const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc)); + const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size; +@@ -2164,7 +2313,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + fattn_kernel_t fattn_kernel; + if (logit_softcap == 0.0f) { + constexpr bool use_logit_softcap = false; +- fattn_kernel = flash_attn_ext_f16; ++ fattn_kernel = flash_attn_ext_f16; // opencoti-hook: WS1 turbo-MMA (#628) + + #if !defined(GGML_USE_MUSA) + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; +@@ -2175,7 +2324,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + #endif // !defined(GGML_USE_MUSA) + } else { + constexpr bool use_logit_softcap = true; +- fattn_kernel = flash_attn_ext_f16; ++ fattn_kernel = flash_attn_ext_f16; // opencoti-hook: WS1 turbo-MMA (#628) + + #if !defined(GGML_USE_MUSA) + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; +@@ -2186,8 +2335,11 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + #endif // !defined(GGML_USE_MUSA) + } + ++ // opencoti-hook: WS1 turbo-MMA (#628): for quantized K/V DO NOT pre-convert to f16 (need_f16_*=false) — ++ // the kernel dequants the raw q8_0/turbo blocks into the smem tile. f16 keeps need_f16_*=true (no-op / ++ // byte-identical: K,V are already f16). stream_k unchanged. + launch_fattn +- (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host); ++ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, !is_quant_kv, !is_quant_kv, true, warp_size_host); + } + + +@@ -2589,6 +2741,11 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + template void ggml_cuda_flash_attn_ext_mma_f16_case \ + (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ + ++// opencoti-hook: WS1 turbo-MMA (#628) — explicit-instantiation macro for the scalar-K/turbo-V MMA case. ++#define DECL_FATTN_MMA_F16_TURBO_CASE(DKQ, DV, ncols1, ncols2, TYPE_K, TYPE_V) \ ++ template void ggml_cuda_flash_attn_ext_mma_f16_case \ ++ (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ ++ + #define DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(DKQ, DV, ncols) \ + extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 1, 1); \ + extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 2, 2); \ +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index 6fc9679..a709dbe 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2146,12 +2146,71 @@ ggml_tensor * llm_graph_context::build_attn_mha( + (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) && + tbv_nq_per_strm <= tbv_fuse_max_nq; // opencoti #533: TCQ de-fuses at prefill (now has a materialize-lift) + +- if (fused_turbo) { +- // forward-rotate Q (×scale_inv → signs₁ → WHT → 1/√G·signs₂); group auto-picks 128 since +- // head_dim%128==0 for both 128 and 256. Q reaches here as the contiguous projection reshape, +- // but guard defensively (a rope'd view would trip the op's contiguity assert). +- if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } +- q = ggml_turbo_wht(ctx0, q, /*direction=*/0, /*group_size=*/0, /*scale=*/nullptr); ++ // opencoti-hook: turbo high-bit scalar-K (WS1/#628). Asymmetric in-register fuse: strong scalar K ++ // (q4_0/q5_0/q6_0/q8_0, true-space, read by the normal FA-VEC scalar path) + cheap turbo/TCQ V ++ // (FWHT-rotated, read in-register — no f16 materialize). The two Parseval identities are INDEPENDENT: ++ // the KQ pair WHT(Q⊘s)·K_stored==Q_true·K_true needs the forward-Q-WHT ONLY when K is rotated; the ++ // V→O pair WHT⁻¹(Σp·V_stored)⊘s==Σp·V_true needs the output inverse-WHT ONLY when V is rotated. For ++ // scalar-K + turbo-V: K is true-space → NO forward-Q-WHT (wht_q=false); V is rotated → output ++ // inverse-WHT still applies (wht_o=true). K bit-width > V-bit-width; reverse (turbo-K/scalar-V) NOT ++ // supported. D512 asym has no vec instance yet → keeps the else-branch materialize. De-fuses at prefill. ++ const auto is_scalar_vec_k = [](enum ggml_type t) { ++ return t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q5_0 || t == GGML_TYPE_Q6_0 || t == GGML_TYPE_Q8_0; ++ }; ++ // V restricted to EXACTLY the built asym instance set (WS1/#628) — not the broader is_turbo_vec, so a ++ // scalar-K + turbo4_0-V config (no instance built) de-fuses to the materialize path instead of aborting. ++ const auto is_built_turbo_v = [](enum ggml_type t) { ++ return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || ++ t == GGML_TYPE_TURBO2_TCQ || t == GGML_TYPE_TURBO3_TCQ; ++ }; ++ // opencoti-hook: WS1 turbo-MMA (#628/#630) — head-dim-split OPT-IN knobs (default OFF). The graph ++ // ALWAYS fuses D256 (scalar-K/turbo-V) to the FUSED-VEC path (pre-#628 shipped behavior = the safe, ++ // validated default); the CUDA selector (fattn.cu) only re-routes the D256 VERIFY batch to the ++ // turbo-MMA kernel when WS1_TURBO_MMA_D256=1. D512 asym-fuse stays behind WS1_TURBO_MMA_D512. ++ // #630 VERDICT (do NOT default-on): turbo-MMA net vs materialize = +4.2% @32k, +7.6% @128k, but ++ // −13.4% @256k — a SESSION-context crossover (128kne[1] can't observe session ctx ++ // (a k->ne[1]1) when the D512 turbo-MMA is enabled, so those long-ctx-dominant ++ // layers read turbo in-register (+ wht_o) instead of the per-forward O(n_kv) materialize cast. n_q=1 ++ // decode keeps head_dim ∈ {128,256} only ⇒ D512 decode stays else-branch materialize (no D512 asym VEC). ++ const bool fused_scalar_k_turbo_v = ++ cparams.flash_attn && kq_b == nullptr && ++ is_scalar_vec_k(k->type) && is_built_turbo_v(v->type) && ++ (k->ne[0] == 128 || k->ne[0] == 256 || ++ (k->ne[0] == 512 && ws1_turbo_mma_d512 && tbv_nq_per_strm > 1)) && ++ tbv_nq_per_strm <= tbv_fuse_max_nq; ++ ++ // wht_q: forward-rotate Q (K is a rotated turbo type). wht_o: inverse-rotate the FA output (V is a ++ // rotated turbo type). Symmetric turbo → both; asymmetric scalar-K/turbo-V → wht_o only. fused_any ++ // is the in-register (no-materialize) gate; the else-branch below still materializes turbo→f16. ++ // opencoti-hook: WS1 turbo-MMA (#628) de-rotate — when the derotate-MMA verify kernel runs (WS1_DEROTATE ++ // on + WS1_TURBO_MMA on + the D256 q8_0/turbo2 verify shape), the CUDA kernel inverse-FWHTs V in-load, so ++ // the FA output is ALREADY true-space; the graph MUST skip wht_o (else double de-rotate = wrong). This ++ // mirrors the CUDA selector/dispatch routing (fattn.cu) EXACTLY; tbv_nq_per_strm==FA-op Q->ne[1] at ++ // n_stream==1 (the gate config). Decode (n_q=1 → VEC, rotated-V) keeps wht_o. Default-off ⇒ byte-identical. ++ // (D512 verify keeps wht_o = rotated-MMA — ws1_derotate_mma_active is D256-only, de-rotate is rejected.) ++ const bool ws1_derotate_mma_active = ws1_turbo_mma_d256 && ws1_derotate_on && ++ fused_scalar_k_turbo_v && k->ne[0] == 256 && ++ k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_TURBO2_0 && tbv_nq_per_strm > 1; ++ const bool wht_q = fused_turbo; ++ const bool wht_o = (fused_turbo || fused_scalar_k_turbo_v) && !ws1_derotate_mma_active; ++ const bool fused_any = fused_turbo || fused_scalar_k_turbo_v; ++ ++ if (fused_any) { ++ if (wht_q) { ++ // forward-rotate Q (×scale_inv → signs₁ → WHT → 1/√G·signs₂); group auto-picks 128 since ++ // head_dim%128==0 for both 128 and 256. Q reaches here as the contiguous projection reshape, ++ // but guard defensively (a rope'd view would trip the op's contiguity assert). ++ if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } ++ q = ggml_turbo_wht(ctx0, q, /*direction=*/0, /*group_size=*/0, /*scale=*/nullptr); ++ } + } else { + const auto is_turbo = [](enum ggml_type t) { + return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_TURBO8_0 +@@ -2229,11 +2288,13 @@ ggml_tensor * llm_graph_context::build_attn_mha( + opencoti_sparse_block_sel = nullptr; + } + +- if (fused_turbo) { ++ if (wht_o) { + // opencoti-hook: turboquant perf-lift (M6-S2 Level B). Inverse-WHT + InnerQ un-equalize + // on the SMALL FA output (signs₂ → WHT → 1/√G·signs₁ → ×scale_inv), recovering true-space +- // attention from the rotated/equalized cache. ne[0]=head_dim_v keys the InnerQ slot; for +- // turbo K==V the per-width scale matches the one folded into the forward-Q rotation above. ++ // attention from the rotated/equalized cache. ne[0]=head_dim_v keys the InnerQ slot. This is ++ // a purely V-side transform (the V→O Parseval pair), so it applies whenever V is a rotated ++ // turbo type — for symmetric turbo the scale matches the forward-Q rotation, and for the ++ // asymmetric scalar-K/turbo-V fuse (WS1/#628) it stands alone (there is no forward-Q-WHT). + cur = ggml_turbo_wht(ctx0, cur, /*direction=*/1, /*group_size=*/0, /*scale=*/nullptr); + } + +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +index f3ed20e..c60a5ba 100644 +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -193,6 +193,34 @@ collect_gpu_sources() { + done + fi + ++ # 3e. opencoti-hook: turbo high-bit scalar-K (WS1/#628). Asymmetric strong scalar-K (q4_0/q5_0/q6_0/ ++ # q8_0) + cheap turbo/TCQ-V FA-VEC instances — read K as scalar + V in-register as turbo/TCQ (no ++ # f16 materialize). Like 3b/3d these are FUSED-decode instances that MUST compile in the DEFAULT ++ # build (the graph emits scalar-K/turbo-V to FA at decode; without them the DSO lacks ++ # ggml_cuda_flash_attn_ext_vec_case and dlopen fails when that KV first reaches ++ # flash-attention). No-op under FA_ALL_QUANTS=1 (already globbed) + vanilla tree ([ -f ] guard). ++ # Design invariant: K bit-width > V-bit-width; reverse (turbo-K/scalar-V) intentionally NOT built. ++ if [ "$fa_all_quants" != "1" ]; then ++ for f in "$ti_dir"/fattn-vec-instance-q8_0-turbo2_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q8_0-turbo3_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q8_0-turbo2_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q8_0-turbo3_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-turbo2_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-turbo3_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-turbo2_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q6_0-turbo3_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q5_0-turbo2_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q5_0-turbo3_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q5_0-turbo2_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q5_0-turbo3_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q4_0-turbo2_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q4_0-turbo3_0.cu \ ++ "$ti_dir"/fattn-vec-instance-q4_0-turbo2_tcq.cu \ ++ "$ti_dir"/fattn-vec-instance-q4_0-turbo3_tcq.cu; do ++ [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" ++ done ++ fi ++ + # 4. mmf instances (always included) + for f in "$ti_dir"/mmf-*.cu; do + [ -f "$f" ] && CUDA_SOURCES="$CUDA_SOURCES $f" +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2.cu +new file mode 100644 +index 0000000..95ca8a1 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2.cu +@@ -0,0 +1,90 @@ ++// opencoti-hook: WS1 turbo-MMA (#628, bug-2126) — scalar-K (q8_0) + turbo-V (turbo2_0) tensor-core ++// VERIFY kernel. Registered as a NEW additive TU (auto-collected by cuda.sh's fattn-mma-*.cu glob). ++// ++// Beachhead: D256 (Gemma SWA/local head_dim), asym -ctk q8_0 -ctv turbo2, on the DCA single_chunk / ++// stock ggml_flash_attn_ext path. The graph (build_attn_mha, llama-graph.cpp) emits turbo-V blocks + ++// q8_0-K to the fused FA op for verify batches (tbv_nq_per_strm <= tbv_fuse_max_nq, default 8) and ++// wraps the OUTPUT with wht_o (inverse-FWHT); K stays unrotated (q8_0 true-space, wht_q=false). This ++// dispatch routes those verify batches (n_q>1) to the type-parametrized MMA case ++// (ggml_cuda_flash_attn_ext_mma_f16_case<256,256,ncols1,ncols2, Q8_0, TURBO2_0>), which decodes ++// K->plain-f16 and V->rotated-f16 into the MMA smem tile (flash_attn_ext_f16_load_tile_dequant) and ++// runs tensor cores. The graph wht_o makes it exact in true space (logit-equiv gate, real_frac≈0). ++// ++// The ncols1/ncols2 selection mirrors ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2/ncols1 in fattn.cu ++// verbatim (fixed to DKQ==DV==256) so the tile shapes match the validated f16 path 1:1. ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_turbo_q8_0_turbo2_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++} ++ ++// Entry point invoked from fattn.cu's MMA dispatch (ggml_cuda_flash_attn_ext_mma_f16) when the selector ++// routes a scalar-K/turbo-V verify batch (D256, q8_0 K, turbo2 V) to BEST_FATTN_KERNEL_MMA_F16. ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ // opencoti-hook: WS1 turbo-MMA (#628) — push the WS1_DEROTATE knob into this TU's __constant__ once. ++ // 1 ⇒ the kernel inverse-FWHTs turbo-V to true space in-load (graph drops wht_o); 0 ⇒ rotated-V + wht_o. ++ // Set here (turbo TU) so cudaMemcpyToSymbol reaches this TU's d_ws1_derotate copy that the turbo kernel reads. ++ static bool derotate_pushed = false; ++ if (!derotate_pushed) { ++ const char * e = getenv("WS1_DEROTATE"); ++ const int v = (e && e[0] == '1') ? 1 : 0; ++ CUDA_CHECK(cudaMemcpyToSymbol(d_ws1_derotate, &v, sizeof(int))); ++ derotate_pushed = true; ++ } ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d128.cu +new file mode 100644 +index 0000000..db27202 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d128.cu +@@ -0,0 +1,74 @@ ++// opencoti-hook: WS1 turbo-MMA (#630) — D128 generalization of the scalar-K (q8_0) + turbo-V (turbo2_0) ++// tensor-core VERIFY kernel. Qwen-family MoE NextN targets (omni27, qw35a3) use head_dim 128 with FULL ++// attention (no SWA window), so EVERY layer sees the full n_kv — unlike Gemma's SWA-windowed D256 local ++// layers. This TU adds the D128 turbo-MMA verify kernel (ROTATED-V + graph wht_o, the proven-keeper ++// approach from the D256 beachhead), routed by the CUDA selector (fattn.cu) only when WS1_TURBO_MMA_D128=1. ++// New additive TU, auto-collected by cuda.sh's fattn-mma-*.cu glob. Default-off; f16 instances untouched. ++// ++// The ncols1/ncols2 selection mirrors ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2/ncols1 (fattn.cu) ++// for DKQ==DV==128 (full gqa coverage 1/2/4/8, like the D256 beachhead). Rotated V; d_ws1_derotate is ++// never set here (de-rotate is the rejected D256-only path). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_turbo_q8_0_turbo2_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/turbo-V verify batch. ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d512.cu +new file mode 100644 +index 0000000..2e9bf8c +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-turbo-q8_0-turbo2-d512.cu +@@ -0,0 +1,72 @@ ++// opencoti-hook: WS1 turbo-MMA (#628) — D512 generalization of the scalar-K (q8_0) + turbo-V (turbo2_0) ++// tensor-core VERIFY kernel. Gemma-4 GLOBAL layers (head_dim 512, full-KV) dominate long-ctx decode; this ++// replaces their per-forward turbo→f16 materialize with a direct dequant-into-the-MMA-tile verify kernel ++// (ROTATED-V + graph wht_o, the proven-keeper approach from the D256 beachhead — NOT de-rotate). Kept for ++// n_q>1 verify only; n_q=1 decode stays materialize (no D512 asym VEC instance). New additive TU, sharded ++// from the D256 TU so the heavy D512 kernels compile in parallel. Auto-collected by cuda.sh fattn-mma-*.cu. ++// ++// The ncols1/ncols2 selection mirrors ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2/ncols1 (fattn.cu) ++// for DKQ==DV==512 (non-Volta): D512 has no gqa<=2 MMA instance (bug-567) so only gqa_ratio>2 (ncols2∈{4,8}) ++// is reachable — A4B/12B/31B global layers are gqa_ratio>=4. Rotated V; d_ws1_derotate is never set here. ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_turbo_q8_0_turbo2_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/turbo-V verify batch. ++void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ // D512 (DKQ>256): only gqa_ratio>2 has an MMA instance (bug-567); the graph only fuses D512 turbo when ++ // the verify shape will reach here, and A4B/12B/31B global layers are gqa_ratio>=4. ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_turbo_q8_0_turbo2_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ GGML_ABORT("WS1 turbo-MMA D512 (#628): no MMA instance for gqa_ratio<=2"); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_0.cu +new file mode 100644 +index 0000000..d13f386 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q4_0, higher bit) + cheap compressed V (turbo2_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_tcq.cu +new file mode 100644 +index 0000000..1358528 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo2_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q4_0, higher bit) + cheap compressed V (turbo2_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_0.cu +new file mode 100644 +index 0000000..b0f8db2 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q4_0, higher bit) + cheap compressed V (turbo3_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_tcq.cu +new file mode 100644 +index 0000000..d4d49b2 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-turbo3_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q4_0, higher bit) + cheap compressed V (turbo3_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_0.cu +new file mode 100644 +index 0000000..59b972f +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q5_0, higher bit) + cheap compressed V (turbo2_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_tcq.cu +new file mode 100644 +index 0000000..5b008e7 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo2_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q5_0, higher bit) + cheap compressed V (turbo2_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_0.cu +new file mode 100644 +index 0000000..79a2225 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q5_0, higher bit) + cheap compressed V (turbo3_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_tcq.cu +new file mode 100644 +index 0000000..b5b2c21 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-turbo3_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q5_0, higher bit) + cheap compressed V (turbo3_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_0.cu +new file mode 100644 +index 0000000..8818957 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q6_0, higher bit) + cheap compressed V (turbo2_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_tcq.cu +new file mode 100644 +index 0000000..951d413 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo2_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q6_0, higher bit) + cheap compressed V (turbo2_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_0.cu +new file mode 100644 +index 0000000..73226c7 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q6_0, higher bit) + cheap compressed V (turbo3_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_tcq.cu +new file mode 100644 +index 0000000..52b4269 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q6_0-turbo3_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q6_0, higher bit) + cheap compressed V (turbo3_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q6_0, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +new file mode 100644 +index 0000000..be25333 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q8_0, higher bit) + cheap compressed V (turbo2_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_tcq.cu +new file mode 100644 +index 0000000..ba30020 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q8_0, higher bit) + cheap compressed V (turbo2_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +new file mode 100644 +index 0000000..45217ba +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q8_0, higher bit) + cheap compressed V (turbo3_0), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_tcq.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_tcq.cu +new file mode 100644 +index 0000000..a028120 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_tcq.cu +@@ -0,0 +1,13 @@ ++// opencoti-hook: turbo high-bit scalar-K (WS1/#628) — asymmetric scalar-K + turbo/TCQ-V FA-VEC cell. ++// Strong scalar K (q8_0, higher bit) + cheap compressed V (turbo3_tcq), read fully in-register (K via the ++// scalar vec_dot_KQ, V via the turbo/TCQ dequantize_V) — no f16 materialization. K stays true-space so ++// the graph applies NO forward-WHT to Q; V is FWHT-rotated so the graph applies the OUTPUT inverse-WHT ++// only (build_attn_mha wht_q/wht_o split). Design invariant: K bit-width > V bit-width; reverse NOT added. ++// tcq-V TUs use the baked default d_turbo{2,3}_tcq_codebook_fattn (correct for production; A' override ++// via TURBO_TCQ_CB{,2} does not reach these TUs — acceptable, ships default). Mirrors fattn-vec-instance-q6_0-q4_0.cu. ++ ++#include "../fattn-vec.cuh" ++ ++DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_TCQ); diff --git a/patches/0108-ws2-dca-fused-turbo-v-inregister.patch b/patches/0108-ws2-dca-fused-turbo-v-inregister.patch new file mode 100644 index 0000000000000000000000000000000000000000..80c16e13a1cdf16028c2c8a344e6b4ececf1f7ff --- /dev/null +++ b/patches/0108-ws2-dca-fused-turbo-v-inregister.patch @@ -0,0 +1,960 @@ +opencoti patch 0108 — WS2 turbo-V in-register DCA fused decode (bug-2134 / #629) + +WS2's decode lever for the multichunk DCA fused kernel: read a rotated turbo/TCQ +V cache IN-REGISTER instead of casting the whole O(n_kv) V cache to f16 every +decode step. The cpy_turbo2_f16 whole-cache cast grew with context and dominated +long-ctx decode; reading V in-register removes it. Now DEFAULT-ON (escape hatch +`WS2_DCA_TURBO_V=0` forces the legacy f16-materialize path). + +dca.cpp (build_attn_dca multichunk): gate `dca_v_inreg_turbo` fires only when the +V tensor is a serving turbo tier (turbo2_0 / turbo3_0 / turbo2_tcq / turbo3_tcq) +at head_dim 512 (Gemma-4 global) or 256 (omni27 / qw35a3), not V-transposed, and +— S3a n_q gate (PURE win) — the ubatch query count `q_cur->ne[2] <= WS2_NQ_MAX` +(default 16). In-register turbo-V wins big at DECODE (small n_q: +54/+89/+127% @ +128/256/512k) but is SLOWER at PREFILL (large n_q: −8..17%, dca_fused +42%/call), +so large-n_q ubatches (and the reserve worst-case) keep the safe f16 path. The +graph applies the same OUTPUT inverse-WHT (`wht_o`, dir=1) that de-rotates every +tier — no extra graph nodes vs the f16 path. + +Kernel side: new D512 + D256 turbo/TCQ dca_fused MMA instance macros +(fattn-mma-f16.cuh) + the D256 turbo/tcq V-type dispatch case in fattn.cu, plus +32 new template-instance TUs (fattn-mma-f16-dca-fused-turbo{,2tcq,3,3tcq}{,-d256} +-instance-{0..3}.cu — F16-K × turbo{2,3}_{0,tcq}-V, D512 and D256, sharded 4 +configs/TU to parallelize the nvcc tail). The shards carry only +`DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(...)` decls and are auto-collected by +cuda.sh's `fattn-mma-*.cu` glob (no build-wiring change). fattn-mma-f16.cuh also +exempts the new turbo/TCQ dca_fused instances from the f16-only V-type assert. + +Off-path (symmetric / non-DCA / f16-V / large-n_q) is byte-identical. Invariant: +K bit-width ≥ V; reverse not built. VALIDATED (#629, WS2): decode win growing +with ctx (+89% @256k, +127% @512k), lossless (teacher-forced TV 0.0009, niah 100). +DSO change (needs a cuda rebuild). Provenance: opencoti-original. **Applies after +`0107`.** + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +index bd94bfb..e618015 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -1099,7 +1099,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( + constexpr bool use_cp_async = nstages == 1; + if constexpr (type_V != GGML_TYPE_F16) { // opencoti-hook: WS1 turbo-MMA (#628) + // stride_V carries BYTES for quant V; i0_start is a logical element (D) offset. +- flash_attn_ext_f16_load_tile_dequant ++ // opencoti-hook: WS2 turbo-V DCA (#629) FIX A — the dequant V loader is the manual ++ // (non-cp_async) path, so it can carry an oob check even while the cp_async K/mask loads ++ // keep oob_check=false (they static_assert !oob_check). Force the bound ON for a turbo V ++ // under dca_fused so the compact turbo cache is never read past ne11 (k_VKQ_sup is the ++ // real per-block bound at the boundary tile). Non-fused / f16 keep the original oob_check. ++ constexpr bool oob_check_v = oob_check || (dca_fused && type_V != GGML_TYPE_F16); ++ flash_attn_ext_f16_load_tile_dequant + ((const char *) V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, i0_diff/2, stride_V, k_VKQ_sup, i0_start); + } else { + flash_attn_ext_f16_load_tile +@@ -1431,6 +1437,15 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + // last_iter is dead at nstages<=1 (used only under nstages>1) ⇒ pass false everywhere; per-block + // k_VKQ_sup: full nbatch_fa for the whole-block INTER/SUCC bands, real ne11-bound for INTRA's tail. + if constexpr (dca_fused) { ++ // opencoti-hook: WS2 turbo-V DCA (#629) FIX A — the analytical phases below read full nbatch_fa ++ // V-tiles with oob_check=false, deliberately over-reading past ne11 (the INTRA mask / band bounds ++ // zero the surplus). That is safe on the f16-lift V (slack-padded pool buffer) but FAULTS on the ++ // exact-sized compact turbo cache read in-register. For a turbo type_V ONLY, force oob_check=true so ++ // load_tile_dequant honours inbatch_fa ⇒ no rows masked ⇒ same result; boundary block: trims the tail read). f16 keeps ++ // oob_check=false / k_VKQ_sup=nbatch_fa ⇒ BYTE-IDENTICAL. Result is lossless either way (surplus rows ++ // contribute zero: f16 reads-then-masks, turbo skips-reading). ++ constexpr bool oob_v = (type_V != GGML_TYPE_F16); + if (mask_inter_h == nullptr) { + // ===== CONTIGUOUS FAST PATH (cell index == absolute position) — UNCHANGED from #617/#618 ===== + // Stream-k: this CUDA block owns key-blocks [kb0_start, kb0_stop) of the output tile. The three DCA +@@ -1465,10 +1480,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = intra_lo; kb0 < intra_hi; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + } + if (succ_hi > succ_lo) { // PHASE SUCC (immediately-prior chunk, Q_succ, band-only — no mask) +@@ -1478,10 +1493,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = succ_lo; kb0 < succ_hi; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_succ_f2, K_h2, V_h2, nullptr, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + } + if (inter_hi > inter_lo) { // PHASE INTER (older chunks, Q_inter, band-only — no mask) +@@ -1491,10 +1506,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = inter_lo; kb0 < inter_hi; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_inter_f2, K_h2, V_h2, nullptr, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + } + } else { +@@ -1519,10 +1534,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + // PHASE SUCC (Q_succ, dense mask_succ) + __syncthreads(); +@@ -1531,10 +1546,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_succ_f2, K_h2, V_h2, mask_succ_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + // PHASE INTER (Q_inter, dense mask_inter) + __syncthreads(); +@@ -1543,10 +1558,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + for (int kb0 = kb0_start; kb0 < kb0_stop; ++kb0) { + flash_attn_ext_f16_iter + ++ T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, /*dca_fused=*/true, type_K, type_V> + (Q_inter_f2, K_h2, V_h2, mask_inter_h, dstk, dstk_fixup, scale, slope, logit_softcap, + ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, +- KQ_max, KQ_rowsum, jt, kb0, nbatch_fa); ++ KQ_max, KQ_rowsum, jt, kb0, (oob_v ? int(ne11 - kb0*nbatch_fa) : nbatch_fa)); + } + } + } +@@ -2348,7 +2363,11 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml + // the partitioned causal range in INTER→SUCC→INTRA phases via process_tile<...,dca_fused=true> which + // clips each phase to the block's owned [kb0_start,kb0_stop). Extra params vs the stock kernel: Q_succ/ + // Q_inter (pre-roped Q for the older-chunk phases), pos_q (I32 real abs query pos), chunk (tokens). +-template ++// opencoti-hook: WS2 turbo-V DCA (#629) — type_K/type_V thread the in-register dequant (load_tile_dequant) ++// into the multichunk DCA fused kernel, killing the whole-cache dca_lift_to_f16 (cpy_turbo2_f16) cast. ++// Default F16/F16 keeps the shipped f16 path byte-identical. ++template + __launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2)) + static __global__ void flash_attn_ext_f16_dca_fused( + const char * __restrict__ Q, +@@ -2391,9 +2410,13 @@ static __global__ void flash_attn_ext_f16_dca_fused( + const int gqa_ratio = ne02 / ne12; + const int stride_Q1 = nb01 / sizeof(float2); + const int stride_Q2 = nb02 / sizeof(float2); +- const int stride_K = nb11 / sizeof(half2); ++ // opencoti-hook: WS2 turbo-V DCA (#629) — type-aware K/V strides, matching the standard kernel entry ++ // (2152/2155): load_tile_dequant reads stride in HALF2-elements for f16 but in BYTES for a quant/turbo ++ // cache. The pre-WS2 dca_fused always materialized quant→f16 so /sizeof(half2) was unconditionally ++ // right; the in-register turbo2 V path needs the raw byte stride or the vectorized load misaligns. ++ const int stride_K = (type_K == GGML_TYPE_F16) ? int(nb11 / sizeof(half2)) : int(nb11); + const int stride_mask = nb31 / sizeof(half); +- const int stride_V = V_is_K_view ? stride_K : nb21 / sizeof(half2); ++ const int stride_V = V_is_K_view ? stride_K : ((type_V == GGML_TYPE_F16) ? int(nb21 / sizeof(half2)) : int(nb21)); + + const int iter_k = (ne11 + (nbatch_fa - 1)) / nbatch_fa; + const int iter_j = (int(ne01.z) + (ncols1 - 1)) / ncols1; +@@ -2442,13 +2465,13 @@ static __global__ void flash_attn_ext_f16_dca_fused( + constexpr bool is_fixup = false; + if (kb0_start == 0) { + constexpr bool needs_fixup = false; // block works on an entire tile +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, + kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); + } else { + constexpr bool needs_fixup = true; // block is missing the beginning of a tile +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, + kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); +@@ -2486,7 +2509,7 @@ static __global__ void flash_attn_ext_f16_dca_fused( + const float slope = 1.0f; + constexpr bool is_fixup = true; // last index writes to the fixup buffer to avoid races + constexpr bool needs_fixup = false; +- flash_attn_ext_f16_process_tile ++ flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, nullptr, dstk, dst_meta, scale, slope, logit_softcap, + ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, + kb0_start, kb0_stop, /*kb0_min=*/0, Q_succ_f2, Q_inter_f2, dca_cq, dca_c_blk, mask_succ_h, mask_inter_h); +@@ -2503,7 +2526,13 @@ static __global__ void flash_attn_ext_f16_dca_fused( + // src[5]=Q_succ, src[6]=Q_inter). Mirrors launch_fattn: K/V already f16 (DCA), stream-k decomposition + + // dst_tmp_meta partials + fixup (dst_lse=nullptr — fused needs no LSE out), KV_max from the INTRA mask + // gives the per-tile causal bound. No parallel-blocks path (stream-k always covers the triangle). +-template ++// opencoti-hook: WS2 turbo-V DCA (#629) — type_K/type_V default F16 (byte-identical shipped path). When ++// type_V is a turbo type the V-side in-launch to_fp16_nc materialize is SKIPPED and the raw turbo cache is ++// handed to the kernel (load_tile_dequant reads it in-register, rotated), killing the whole-cache ++// dca_lift_to_f16 (cpy_turbo2_f16) cast. The graph applies the output inverse-FWHT (wht_o) since V stays ++// rotated. K keeps type_K=F16 (its cheap in-launch to_fp16_nc convert) for the beachhead. ++template + void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; +@@ -2520,9 +2549,14 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + // buffer, mirroring launch_fattn) instead of being whole-cache materialized to f16 in the graph every + // forward (the prefill cliff: ~138 vs ~1212 tps at 98k/q5_0). Accept f16, or any type with a to_fp16_nc + // converter; reject anything else (the kernel would have no f16 source). ++ // opencoti-hook: WS2 turbo-V DCA (#629, S3b) — V is materialized via to_fp16_nc ONLY when type_V==F16 ++ // (the shipped lift path). An in-register turbo/TCQ V (type_V != F16) is read raw by load_tile_dequant / ++ // get_dequantize_V and never needs to_fp16_nc (TCQ has no to_fp16_nc — it lifts via cpy_turbo*_tcq_f16), ++ // so exempt it from the converter requirement. K is always materialized when quantized, so it keeps the ++ // converter requirement unconditionally. + GGML_ASSERT((K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) && +- (V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) && +- "dca fused: K/V must be f16 or have a to_fp16_nc converter"); ++ (type_V != GGML_TYPE_F16 || V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) && ++ "dca fused: K must be f16 or have a to_fp16_nc converter; a materialized (type_V==F16) V likewise"); + GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors"); + // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER + // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent. +@@ -2597,7 +2631,11 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + nb_k2 = K->ne[1] * nb_k1; + nb_k3 = K->ne[2] * nb_k2; + } +- if (V->type != GGML_TYPE_F16) { ++ // opencoti-hook: WS2 turbo-V DCA (#629) — when the kernel is instantiated for a turbo type_V it reads ++ // the raw turbo cache in-register (load_tile_dequant, rotated); SKIP the whole-cache to_fp16_nc ++ // materialize (v_data/nb_v* keep the raw turbo pointer + block strides set above). type_V==F16 keeps ++ // the shipped materialize path byte-identical. ++ if (type_V == GGML_TYPE_F16 && V->type != GGML_TYPE_F16) { + const size_t ts = ggml_type_size(V->type); + GGML_ASSERT(V->nb[0] == ts && "dca fused: quant V must have contiguous dim-0 (block axis)"); + to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(V->type); +@@ -2731,9 +2769,9 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + } + }; + if (logit_softcap == 0.0f) { +- launch(flash_attn_ext_f16_dca_fused); ++ launch(flash_attn_ext_f16_dca_fused); + } else { +- launch(flash_attn_ext_f16_dca_fused); ++ launch(flash_attn_ext_f16_dca_fused); + } + } + +@@ -2827,6 +2865,22 @@ extern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32); + extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, ncols1, ncols2); \ + extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, ncols1, ncols2); \ + ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 global turbo2-V in-register dca_fused (F16-K + TURBO2_0-V), ++// the A4B beachhead. Type-parametrized case; defined in the turbo instance TU, referenced by the fattn.cu ++// dca dispatch when WS2_DCA_TURBO_V is on + V is turbo2 at head_dim 512. ++#define DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(DKQ, DV, ncols1, ncols2, TK, TV) \ ++ template void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case \ ++ (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ ++ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — extern the D512 in-register instances for every serving V ++// tier (F16-K + turbo{2,3}_0 / turbo{2,3}_tcq V). One extern per tier per (ncols1,ncols2); the .cu shards ++// instantiate them (sharded 4 configs/TU/tier to bound the nvcc tail). ++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(ncols1, ncols2) \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++ + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 1) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 1) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 1) +@@ -2843,3 +2897,48 @@ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 1, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 8) ++ ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 turbo2-V externs (all ncols configs the dca dispatch reaches). ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 8, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(16, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(32, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(64, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 4, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 8, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(16, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(32, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 2, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 4, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 8, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512(16, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 1, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 2, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 4, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D512( 8, 8) ++ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V externs for the Qwen-arch head_dim ++// (qwen35/qwen35moe global attention, no SWA → DCA-routed at long ctx). Only the _0 tiers are built for ++// D256 (turbo2_0 + turbo3_0, the primary lossless-class); TCQ D256 is a cheap additive follow-up if a ++// Qwen TCQ config is ever served. NOT the Gemma SWA-local D256 (that is never DCA-routed). ++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(ncols1, ncols2) \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, ncols1, ncols2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++ ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 8, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(16, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(32, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(64, 1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 4, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 8, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(16, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(32, 2) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 2, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 4, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 8, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256(16, 4) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 1, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 2, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 4, 8) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 8, 8) +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index 2eb1fc7..29f1a1e 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -821,28 +821,30 @@ static __global__ void streaming_lse_kernel( + // fused case + process_tile are already DKQ-templated (softcap whitelist DKQ∈{128,256,512}); the config + // table has matching entries for all three. The single_chunk path (plain ggml_flash_attn_ext) was never + // affected — only the analytical-bands fused dispatch went through here. +-template ++// opencoti-hook: WS2 turbo-V DCA (#629) — type_K/type_V (default F16/F16 = byte-identical shipped path) ++// thread the in-register turbo-V read into the dca_fused case selection. ++template + static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const ggml_tensor * Q = dst->src[0]; + if constexpr (ncols2 <= 8) { + if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) { +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; + } + } + if constexpr (ncols2 <= 16) { + if ((turing_mma_available(cc) || amd_wmma_available(cc)) && Q->ne[1] <= 16/ncols2) { +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; + } + } + if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || amd_wmma_available(cc) || Q->ne[1] <= 32/ncols2) { +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); return; + } +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ctx, dst); + } + + // gqa_ratio → ncols2 selection for a fixed head dim DKQ (the old switch body, now DKQ-parameterized). +-template ++template + static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const ggml_tensor * Q = dst->src[0]; +@@ -850,15 +852,15 @@ static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa(ggml_backend + GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); + const int gqa_ratio = Q->ne[2] / K->ne[2]; + if (cc == GGML_CUDA_CC_VOLTA) { +- if (gqa_ratio % 8 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- if (gqa_ratio % 4 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- if (gqa_ratio % 2 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; ++ if (gqa_ratio % 8 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio % 4 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio % 2 == 0) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; + } +- if (gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- if (gqa_ratio > 2) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- if (gqa_ratio > 1) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); ++ if (gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio > 2) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ if (gqa_ratio > 1) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); return; } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch_ncols1(ctx, dst); + } + + // opencoti F5 dca (#623): head-dim switch mirroring the stock ggml_cuda_flash_attn_ext_mma_f16 path. DCA +@@ -872,10 +874,33 @@ static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ggml_backend_cuda_ + ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<128>(ctx, dst); break; + case 256: + GGML_ASSERT(V->ne[0] == 256); +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256>(ctx, dst); break; ++ // opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL head_dim is the Qwen-arch case ++ // (qwen35/qwen35moe, no SWA → DCA-routed). A raw turbo{2,3}_0 V deferred here (WS2 on) routes to ++ // the matching in-register D256 instance; otherwise V was lifted to f16 → shipped f16 path. Only ++ // the _0 tiers are built for D256; a TCQ-V never reaches here in-register (host gate excludes it). ++ switch (V->type) { ++ case GGML_TYPE_TURBO2_0: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256, GGML_TYPE_F16, GGML_TYPE_TURBO2_0>(ctx, dst); break; ++ case GGML_TYPE_TURBO3_0: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256, GGML_TYPE_F16, GGML_TYPE_TURBO3_0>(ctx, dst); break; ++ case GGML_TYPE_TURBO2_TCQ: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ>(ctx, dst); break; ++ case GGML_TYPE_TURBO3_TCQ: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ>(ctx, dst); break; ++ default: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<256>(ctx, dst); break; ++ } ++ break; + case 512: + GGML_ASSERT(V->ne[0] == 512); +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512>(ctx, dst); break; ++ // opencoti-hook: WS2 turbo-V DCA (#629, S3b) — a raw turbo/TCQ V cache reaching here (the graph ++ // deferred it, WS2_DCA_TURBO_V on) routes to the matching in-register D512 turbo instance (F16-K + ++ // turboN-V); otherwise V was already lifted to f16 → the shipped f16 path. K stays f16 (beachhead). ++ // All tiers are FWHT-rotated with InnerQ inactive on this path ⇒ the graph wht_o (scale=null) is ++ // the common de-rotate; get_dequantize_V dispatches the per-tier codebook read. ++ switch (V->type) { ++ case GGML_TYPE_TURBO2_0: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512, GGML_TYPE_F16, GGML_TYPE_TURBO2_0>(ctx, dst); break; ++ case GGML_TYPE_TURBO3_0: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512, GGML_TYPE_F16, GGML_TYPE_TURBO3_0>(ctx, dst); break; ++ case GGML_TYPE_TURBO2_TCQ: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ>(ctx, dst); break; ++ case GGML_TYPE_TURBO3_TCQ: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ>(ctx, dst); break; ++ default: ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<512>(ctx, dst); break; ++ } ++ break; + default: + GGML_ABORT("dca fused: unsupported head dim %d (expected 128/256/512)", (int) Q->ne[0]); + break; +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +index ba747bf..4a314e3 100644 +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -281,6 +281,40 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + const bool defer_k = dca_is_inkernel_liftable(k_raw->type); + const bool defer_v = dca_is_inkernel_liftable(v_raw->type) && !v_trans_raw; + ++ // opencoti-hook: WS2 turbo-V DCA (#629) — read a rotated turbo2 V cache IN-REGISTER in the MULTICHUNK ++ // fused kernel (load_tile_dequant), killing the whole-cache dca_lift_to_f16 → cpy_turbo2_f16 cast (the ++ // measured ~13% ctx-growing bottleneck). Beachhead: D512 global (Gemma), analytical (multichunk) path ++ // only, contiguous cache. DEFAULT-ON (#629 shipped, byte-lossless: real_frac 0 / ffa 1.0 / niah 100 ++ // across A4B+31B D512 and omni27+qw35a3 D256, all 4 turbo tiers); set WS2_DCA_TURBO_V=0 to force the ++ // f16-lift path. V stays rotated ⇒ the graph applies the output inverse-FWHT (wht_o) after the fused op ++ // below (Parseval: WHT⁻¹(Σ p·V_stored) == Σ p·V_true; the 3-phase INTER→SUCC→INTRA online-softmax ++ // weights are scalars, FWHT is linear ⇒ it commutes). ++ static const bool ws2_dca_turbo_v = []() { const char * e = getenv("WS2_DCA_TURBO_V"); return (!e || e[0] != '0'); }(); ++ // opencoti-hook: WS2 turbo-V DCA (#629) S3a — n_q gate (PURE win). In-register turbo-V wins big at DECODE ++ // (small n_q: removes the O(n_kv) whole-cache cast; +54/+89/+127% @128/256/512k) but the in-register ++ // dequant + oob-bound MMA is SLOWER at PREFILL (large n_q: −8-17%, dca_fused kernel +42%/call). So gate on ++ // the ubatch query count: in-register only for n_q ≤ WS2_NQ_MAX (decode + small spec-verify); large-n_q ++ // prefill falls to the f16-lift path (BYTE-IDENTICAL to shipped). q_cur is [n_embd_head, n_head, n_tokens] ++ // so ne[2] is the per-ubatch n_q (ne[1]=n_head); the reserve worst-case (large n_q) also takes the safe ++ // f16 path. Default 16 (covers decode n_q=1 + small spec-verify batches). ++ static const int ws2_nq_max = []() { const char * e = getenv("WS2_NQ_MAX"); return e ? atoi(e) : 16; }(); ++ // S3b — generalize across serving V tiers. All are FWHT-rotated with InnerQ inactive on the fused path, ++ // so the SAME wht_o (dir=1, scale=null) de-rotates every tier (mirrors build_attn_mha's is_turbo_vec set); ++ // get_dequantize_V dispatches the per-tier codebook read. Per-tier correctness is gated by the S3b TV test. ++ const bool ws2_tier_v_0 = v_raw->type == GGML_TYPE_TURBO2_0 || v_raw->type == GGML_TYPE_TURBO3_0; ++ const bool ws2_tier_v_tcq = v_raw->type == GGML_TYPE_TURBO2_TCQ || v_raw->type == GGML_TYPE_TURBO3_TCQ; ++ // opencoti-hook: WS2 turbo-V DCA (#629, S3c) — head_dim×tier admissibility for the in-register read. ++ // D512 (Gemma global) AND D256 (Qwen-arch GLOBAL head_dim — qwen35/qwen35moe, no SWA → DCA-routed at ++ // long ctx): all 4 serving tiers (turbo2_0/turbo3_0/turbo2_tcq/turbo3_tcq) have in-register instances ++ // built + validated. NOT the Gemma SWA-local D256, which is never DCA-routed so never reaches this ++ // multichunk gate. ++ const bool ws2_head_tier_ok = ++ (v_raw->ne[0] == 512 && (ws2_tier_v_0 || ws2_tier_v_tcq)) || ++ (v_raw->ne[0] == 256 && (ws2_tier_v_0 || ws2_tier_v_tcq)); ++ const bool dca_v_inreg_turbo = ws2_dca_turbo_v && !inp_dca->single_chunk && ++ ws2_head_tier_ok && !v_trans_raw && ++ q_cur->ne[2] <= ws2_nq_max; ++ + // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. + // opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16 + // while it is still CONTIGUOUS — a permuted view of a block type is not cleanly cpy-able (the +@@ -293,8 +327,8 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // to_fp16_nc dequants with permuted strides); turbo/TCQ/bf16 → single-pass lift to f16. + k_dca = defer_k ? k_raw : dca_lift_to_f16(ctx0, k_raw); + +- // V: defer scalar-quant straight to the fused op; otherwise lift-while-contiguous then permute. +- if (defer_v) { ++ // V: defer scalar-quant (or WS2 in-register turbo2) straight to the fused op; else lift then permute. ++ if (defer_v || dca_v_inreg_turbo) { + vp = ggml_permute(ctx0, v_raw, 0, 2, 1, 3); + } else { + ggml_tensor * v = v_raw; +@@ -354,6 +388,13 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + const size_t ts = P->nb[0]; + cur = ggml_cont(ctx0, ggml_view_4d(ctx0, P, DV, H, NQ, 1, + DV*ts, DV*H*ts, DV*H*NQ*ts, 0)); ++ // opencoti-hook: WS2 turbo-V DCA (#629) — the fused kernel read V rotated (in-register turbo2), so ++ // its output O is still in rotated space; recover TRUE space with the output inverse-FWHT, exactly ++ // as build_attn_mha's wht_o does for the fused scalar-K/turbo-V decode. turbo2_0 has no InnerQ ⇒ ++ // scale=nullptr. Parseval commutes across the 3-phase online-softmax (weighted sum of V-rows). ++ if (dca_v_inreg_turbo) { ++ cur = ggml_turbo_wht(ctx0, cur, /*direction=*/1, /*group_size=*/0, /*scale=*/nullptr); ++ } + } else { + // opencoti F5 dca (#618): the legacy non-analytical multi-chunk path (3 per-regime + // ggml_flash_attn_ext_lse ops at regime 0 + a host f32 online-softmax combine) was REMOVED. It +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-0.cu +new file mode 100644 +index 000000000..9f32cea86 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-0.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO2_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-1.cu +new file mode 100644 +index 000000000..7e0617140 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-1.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO2_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-2.cu +new file mode 100644 +index 000000000..513d214e9 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-2.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO2_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-3.cu +new file mode 100644 +index 000000000..3c846d664 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-d256-instance-3.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO2_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-0.cu +new file mode 100644 +index 000000000..fe2e36776 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-0.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO2_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-1.cu +new file mode 100644 +index 000000000..a3f9cbda7 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-1.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO2_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-2.cu +new file mode 100644 +index 000000000..f3f440d35 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-2.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO2_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-3.cu +new file mode 100644 +index 000000000..69b180653 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo2tcq-instance-3.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO2_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-0.cu +new file mode 100644 +index 000000000..a0df1460d +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO3_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-1.cu +new file mode 100644 +index 000000000..c09b200a0 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO3_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-2.cu +new file mode 100644 +index 000000000..71c728a25 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO3_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-3.cu +new file mode 100644 +index 000000000..7cb625a2b +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-d256-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO3_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-0.cu +new file mode 100644 +index 000000000..caf3d4b42 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-0.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_0 V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-1.cu +new file mode 100644 +index 000000000..12dbece48 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-1.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_0 V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-2.cu +new file mode 100644 +index 000000000..18c6d6c07 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-2.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_0 V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-3.cu +new file mode 100644 +index 000000000..908794117 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3-instance-3.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_0 V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-0.cu +new file mode 100644 +index 000000000..a9fb65ee6 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-0.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO3_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-1.cu +new file mode 100644 +index 000000000..76c85b8e8 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-1.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO3_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-2.cu +new file mode 100644 +index 000000000..bad438f24 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-2.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO3_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-3.cu +new file mode 100644 +index 000000000..16ed1f9e6 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-d256-instance-3.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL GGML_TYPE_TURBO3_TCQ (F16-K + TCQ-V) in-register ++// multichunk DCA fused instances for the Qwen-arch global head_dim. Sharded 4 configs/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-0.cu +new file mode 100644 +index 000000000..86badd866 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-0.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-1.cu +new file mode 100644 +index 000000000..395693442 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-1.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-2.cu +new file mode 100644 +index 000000000..f0a8f7596 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-2.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-3.cu +new file mode 100644 +index 000000000..68e34f634 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo3tcq-instance-3.cu +@@ -0,0 +1,8 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3b) — D512 GGML_TYPE_TURBO3_TCQ V (F16-K) in-register multichunk ++// DCA fused kernel instances, sharded 4 configs/TU. Auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO3_TCQ); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-0.cu +new file mode 100644 +index 000000000..10026fa41 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO2_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-1.cu +new file mode 100644 +index 000000000..6b58e8461 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO2_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-2.cu +new file mode 100644 +index 000000000..5878e3af7 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO2_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-3.cu +new file mode 100644 +index 000000000..f7c5ae79b +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-d256-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL turbo-V (F16-K + GGML_TYPE_TURBO2_0-V) in-register ++// multichunk DCA fused kernel instances for the Qwen-arch global head_dim (qwen35/qwen35moe, no SWA). ++// NOT the Gemma SWA-local D256 (never DCA-routed). Sharded 4 configs/TU. Externs in ../fattn-mma-f16.cuh. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(256, 256, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-0.cu +new file mode 100644 +index 000000000..9eae260b4 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 global turbo2-V (F16-K + TURBO2_0-V) in-register multichunk ++// DCA fused kernel instances, sharded (4 configs/TU) to parallelize the nvcc tail. Extern decls in ++// ../fattn-mma-f16.cuh; auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 64, 1, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-1.cu +new file mode 100644 +index 000000000..7a072adcf +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 global turbo2-V (F16-K + TURBO2_0-V) in-register multichunk ++// DCA fused kernel instances, sharded (4 configs/TU) to parallelize the nvcc tail. Extern decls in ++// ../fattn-mma-f16.cuh; auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 32, 2, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-2.cu +new file mode 100644 +index 000000000..426bf8174 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 global turbo2-V (F16-K + TURBO2_0-V) in-register multichunk ++// DCA fused kernel instances, sharded (4 configs/TU) to parallelize the nvcc tail. Extern decls in ++// ../fattn-mma-f16.cuh; auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 16, 4, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-3.cu +new file mode 100644 +index 000000000..f4050d427 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-turbo-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: WS2 turbo-V DCA (#629) — D512 global turbo2-V (F16-K + TURBO2_0-V) in-register multichunk ++// DCA fused kernel instances, sharded (4 configs/TU) to parallelize the nvcc tail. Extern decls in ++// ../fattn-mma-f16.cuh; auto-collected by cuda.sh's fattn-mma-*.cu glob. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 1, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 2, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 4, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(512, 512, 8, 8, GGML_TYPE_F16, GGML_TYPE_TURBO2_0); diff --git a/patches/0109-cuda-dso-2gb-fatbin-last.patch b/patches/0109-cuda-dso-2gb-fatbin-last.patch new file mode 100644 index 0000000000000000000000000000000000000000..be7f5353853d683ab19920dd78671522b42cd5c8 --- /dev/null +++ b/patches/0109-cuda-dso-2gb-fatbin-last.patch @@ -0,0 +1,147 @@ +opencoti patch 0109 — CUDA DSO 2GB fatbin-last layout + mcmodel opt-in (bug-2140 / #629) + +The ggml-cuda.so embeds ~1.9GB of .nv_fatbin (6 SM arches × ~260 cubins) +physically BETWEEN low host .text and its high .tdata/.tbss/.got/.plt/.bss. +Adding the WS2 D256 turbo{2,3}_tcq FA instances pushed the image past 2^31, so +intra-module PC32 / TLSGD (thread-local fattn diag vars) / PLT refs must jump the +whole blob → "relocation truncated to fit against .bss" + "PC-relative offset +overflow in PLT entry for launch_fattn" at link. + +PRIMARY fix — cuda-fatbin-last LAYOUT hook (build-functions.sh): an INSERT-AFTER +.bss linker fragment forces .nv_fatbin to the END of the image, so +.text/.rodata/.tdata/.tbss/.got/.plt/.data/.bss cluster in the low ~30MB and +every intra-module reloc is short-range — small code model then suffices and ALL +reloc classes resolve (PC32, TLSGD, PLT), including the TLS ones -mcmodel cannot +reach. The only long ref left is the fatbin-wrapper→.nv_fatbin data pointer, a +64-bit data reloc. Gated `OPENCOTI_FATBIN_LAST` (default 1). Also adds +`-Xlinker=--no-relax` on the nvcc link so the prebuilt crti.o's REX_GOTPCRELX +against __gmon_start__ keeps its 64-bit GOT indirection (the linker's relax-to-PC32 +would overflow; crti.o is a distro small-model prebuilt we cannot recompile). + +OPT-IN fallback — cuda-mcmodel-large (build-functions.sh + cuda.sh): forces the +LARGE code model on ALL host x86 code in the DSO (raw gcc/g++ core-ggml TUs AND +nvcc's cudafe1 device-stubs via --compiler-options, AND at link). Device SASS is +unaffected → zero kernel-perf cost. Now DEFAULT OFF (`OPENCOTI_MCMODEL=none`) +because fatbin-last alone links + dlopens clean; kept as a tunable +(OPENCOTI_MCMODEL=large|medium) fallback. build-functions.sh and cuda.sh read the +same var. + +Host build-script only — no source/kernel change (the DSO relinks with the new +layout). Provenance: opencoti-original. See docs/protocols/BUILD_CYCLE.md. +**Applies after `0108`.** + +diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh +index c60a5ba..fe03d54 100644 +--- a/llamafile/build-functions.sh ++++ b/llamafile/build-functions.sh +@@ -329,6 +329,15 @@ compile_ggml_core() { + -I"$llama_cpp_dir/ggml/src" + ) + ++ # opencoti-hook: cuda-mcmodel-large (WS2 2GB DSO link-wall) — keep the raw gcc/g++ core-ggml TUs ++ # on the SAME code model as the nvcc-compiled TUs, so every object linked into ggml-cuda.so agrees ++ # on host addressing. Device SASS is unaffected. Tunable via OPENCOTI_MCMODEL (default large; set ++ # in cuda.sh). See docs/protocols/BUILD_CYCLE.md. ++ local _mcm="${OPENCOTI_MCMODEL:-none}" ++ if [ -n "$_mcm" ] && [ "$_mcm" != "none" ]; then ++ host_flags+=("-mcmodel=$_mcm") ++ fi ++ + for src in $ggml_core_sources; do + local base=$(basename "$src") + local ext="${base##*.}" +@@ -378,6 +387,20 @@ link_shared_library() { + nvcc) + # nvcc requires -Xcompiler to forward flags to the host driver. + static_cxx_flags="-Xcompiler=-static-libstdc++,-static-libgcc" ++ # opencoti-hook: cuda-mcmodel-large (WS2 2GB DSO link-wall) — large code model at LINK ++ # so the linker-generated PLT/GOT stubs use 64-bit addressing; pairs with the ++ # compile-side flag in cuda.sh. Without this the crti.o __gmon_start__ GOTPCRELX and ++ # launch_fattn PLT entry overflow once .nv_fatbin pushes sections past 2^31. Tunable ++ # via OPENCOTI_MCMODEL (default large). See docs/protocols/BUILD_CYCLE.md. ++ _mcm="${OPENCOTI_MCMODEL:-none}" ++ if [ -n "$_mcm" ] && [ "$_mcm" != "none" ]; then ++ static_cxx_flags="${static_cxx_flags},-mcmodel=$_mcm" ++ fi ++ # crti.o (distro CRT, small-model prebuilt) carries a REX_GOTPCRELX against ++ # undefined __gmon_start__; the linker tries to relax it to a PC32 lea, which ++ # overflows once .nv_fatbin pushes .init→.got past 2^31. --no-relax keeps the ++ # 64-bit GOT indirection. Cannot be fixed by our -mcmodel (crti.o is prebuilt). ++ static_cxx_flags="${static_cxx_flags} -Xlinker=--no-relax" + ;; + *) + # hipcc, g++, clang++ accept these driver flags directly +@@ -386,6 +409,34 @@ link_shared_library() { + esac + fi + ++ # opencoti-hook: cuda-fatbin-last (WS2 2GB DSO link-wall — LAYOUT fix; primary lever, supersedes ++ # -mcmodel). The ~1.9GB .nv_fatbin sits physically between host .text and its ++ # .tdata/.tbss/.got/.plt, so intra-module PC32 / TLSGD (thread-local) / PLT refs must jump the whole ++ # blob → "relocation truncated to fit" + "PC-relative offset overflow in PLT entry" once >2GB. ++ # -mcmodel widens PC32/data relocs to 64-bit but does NOT govern TLS or PLT reach, so it cannot fix ++ # the TLSGD/PLT overflow (opencoti's thread-local fattn diag vars in .tbss/.tdata are referenced by ++ # launch_fattn across the fatbin). Root fix: force .nv_fatbin to the END of the image via an ++ # INSERT-AFTER linker fragment (.nv_fatbin is an orphan section → an explicit rule claims it), so ++ # .text/.rodata/.tdata/.tbss/.got/.plt/.data/.bss cluster in the low ~30MB and every intra-module ++ # reloc is short-range — small code model then suffices; TLS/PLT/PC32 all resolve. The only long ref ++ # left is the fatbin-wrapper→.nv_fatbin data pointer, a 64-bit data reloc (never PC32). Gated ++ # OPENCOTI_FATBIN_LAST (default 1). See docs/protocols/BUILD_CYCLE.md. ++ if [ "${OPENCOTI_FATBIN_LAST:-1}" = "1" ]; then ++ local _fatbin_ld="$build_dir/opencoti-fatbin-last.ld" ++ cat > "$_fatbin_ld" <<'OPENCOTI_LDEOF' ++SECTIONS ++{ ++ .nv_fatbin : { *(.nv_fatbin) } ++} ++INSERT AFTER .bss; ++OPENCOTI_LDEOF ++ case "$(basename "$linker")" in ++ nvcc) static_cxx_flags="$static_cxx_flags -Xlinker=-T -Xlinker=$_fatbin_ld" ;; ++ *) static_cxx_flags="$static_cxx_flags -Wl,-T,$_fatbin_ld" ;; ++ esac ++ echo " fatbin layout: .nv_fatbin forced to end of image (OPENCOTI_FATBIN_LAST=1 — 2GB DSO wall, WS2)" ++ fi ++ + echo "Linking..." + + local obj_files=$(find "$build_dir" -name "*.o" -type f | tr '\n' ' ') +diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh +index 8e38209..09e37cc 100755 +--- a/llamafile/cuda.sh ++++ b/llamafile/cuda.sh +@@ -231,6 +231,25 @@ if [ "$COMPRESS" = "1" ]; then + fi + fi + ++# opencoti-hook: cuda-mcmodel-large (WS2 D256 tcq / 2GB DSO link-wall) — see docs/protocols/BUILD_CYCLE.md. ++# The ggml-cuda.so embeds ~1.9GB of .nv_fatbin (6 SM arches × ~260 cubins) physically between low ++# .text (host code) and high .bss; adding the D256 turbo2/3_tcq FA instances pushed a host ++# device-stub's PC32 reloc to .bss past the 2^31 small-model ceiling → "relocation truncated to fit ++# against .bss" + "PC-relative offset overflow in PLT entry for launch_fattn". Prior bare -mcmodel ++# attempts never reached nvcc's cudafe1.cpp host device-stub pass. Fix: force the LARGE code model on ++# ALL host x86 code in the DSO — it MUST ride inside nvcc's --compiler-options so the device-stubs get ++# it (and at link, build-functions.sh). Device SASS in the fatbin is unaffected → zero kernel-perf ++# cost. NOTE: the PRIMARY fix is the cuda-fatbin-last LAYOUT hook in build-functions.sh (moving ++# .nv_fatbin to the image end fixes ALL reloc classes — PC32, TLSGD, PLT — including the TLS ones ++# -mcmodel CANNOT reach); validated links+dlopens clean with mcmodel OFF. So -mcmodel is now an OPTIONAL ++# fallback, default OFF (none). Tunable: OPENCOTI_MCMODEL=large|medium. build-functions.sh reads the same var. ++OPENCOTI_MCMODEL="${OPENCOTI_MCMODEL:-none}" ++MCMODEL_CFLAG="" ++if [ -n "$OPENCOTI_MCMODEL" ] && [ "$OPENCOTI_MCMODEL" != "none" ]; then ++ MCMODEL_CFLAG=",-mcmodel=$OPENCOTI_MCMODEL" ++ echo " code model: -mcmodel=$OPENCOTI_MCMODEL (host x86 only; 2GB DSO link-wall — WS2)" ++fi ++ + # NVCC compiler flags + # opencoti note: -std=c++17 added because the ggml-cuda .cu sources use + # C++17 features (std::is_same_v, structured bindings, fold expressions). +@@ -245,7 +264,7 @@ COMMON_FLAGS="\ + -I$LLAMA_CPP_DIR/ggml/src \ + -I$GGML_CUDA_DIR \ + --forward-unknown-to-host-compiler \ +- --compiler-options -fPIC,-O2 \ ++ --compiler-options -fPIC,-O2${MCMODEL_CFLAG} \ + -DNDEBUG \ + -DGGML_BUILD=1 \ + -DGGML_SHARED=1 \ diff --git a/patches/0110-ws2-asym-band-nan-guard.patch b/patches/0110-ws2-asym-band-nan-guard.patch new file mode 100644 index 0000000000000000000000000000000000000000..e8ca774fcc960dc88766541b89626b6b632e59af --- /dev/null +++ b/patches/0110-ws2-asym-band-nan-guard.patch @@ -0,0 +1,102 @@ +opencoti patch 0110 — WS2 asym-band NaN guard (bug-2141 / #629 / #626) + +WS2 (#629) reads a turbo-V cache IN-REGISTER inside the multichunk dca_fused +kernel (rotated, FWHT space) and de-rotates the OUTPUT via the graph wht_o +(dca.cpp) instead of materializing V→f16. That output is the online-softmax +normalized attention O = VKQ / rowsum, computed in the shared stream-k combine. + +Under DCA-on decode the fused kernel walks INTER→SUCC→INTRA key-phases under one +continuous online-softmax and load-balances the causal triangle via the stock +kbc stream-k decomposition, so the final divide-by-rowsum lands in the shared +stream-k fixup (fattn-common.cuh flash_attn_stream_k_fixup_uniform/_general) for +seam blocks and in the process_tile complete-tile write-back (fattn-mma-f16.cuh) +otherwise. A DCA band-boundary phase whose mask is all-(−inf) yields a ZERO +exp-sum for that query row (rowsum==0); the plain `dst = VKQ / rowsum` then emits +0/0 = NaN. The graph wht_o's 128-pt butterfly smears that single NaN lane across +all 512 de-rotated output dims → NaN logits → empty generation. + +The fault only surfaces on asymmetric turbo_tcq K≠V (t3tcqK/t2tcqV and +t2tcqK/t3tcqV) at the 256k/8-chunk geometry, where the fused instance pairs a +turbo3-derived f16-lifted K with the WS2 in-register turbo2-V (a composition no +symmetric control exercises): pre-fix niah 20 / 4-of-5 EMPTY at 256k, though 100 +at 512k/16-chunk. Confirmed by the WS2_DCA_TURBO_V=0 discriminator (V→f16-lift, +same K-lift → recovers to niah 100 / 0-empty), isolating the fault to the +in-register-V + wht_o composition, NOT the K-lift/codebook (all per-tensor type +selection verified correct). + +FIX — guard the zero exp-sum divide at all three FA output-normalization sites +that feed the graph wht_o, emitting the identity 0 for a fully-masked row instead +of NaN. The guard preserves the EXACT original division on the rowsum!=0 branch +(not a reciprocal-multiply), so every non-degenerate row — all symmetric turbo/TCQ ++ scalar tiers, and every non-DCA caller of these shared kernels — stays +BIT-IDENTICAL. A fully-masked band contributes 0, the standard flash-attention +identity, so no NaN reaches wht_o. + + - fattn-mma-f16.cuh flash_attn_ext_f16_process_tile (complete-tile write-back) + - fattn-common.cuh flash_attn_stream_k_fixup_uniform (stream-k seam combine) + - fattn-common.cuh flash_attn_stream_k_fixup_general (non-uniform block combine) + +CUDA kernel/header source only; the WS2 in-register decode win is untouched +(the guard is a no-op on the hot path). Provenance: opencoti-original. +See docs/protocols/UPSTREAM_SYNC.md and docs/protocols/BUILD_CYCLE.md. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +index e618015..9d151be 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -2057,8 +2057,20 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( + + if (!needs_fixup && !is_fixup) { + const float KQ_rowsum_j = meta_j[1]; +- dstk_val.x /= KQ_rowsum_j; +- dstk_val.y /= KQ_rowsum_j; ++ // opencoti-hook: WS2 asym-band NaN guard (bug-2141) — a fully-masked DCA ++ // band-boundary phase yields a zero exp-sum (KQ_rowsum_j==0); the plain divide ++ // emits 0/0=NaN which the graph wht_o (dca.cpp) then butterfly-smears across all ++ // de-rotated output dims → NaN logits → empty generation (asym turbo_tcq K!=V ++ // @256k, #629/#626). Emit the identity 0 for a fully-masked row. NO-OP for ++ // rowsum!=0: the exact same division runs → bit-identical for every non-degenerate ++ // case (all symmetric turbo/TCQ + scalar tiers, and non-DCA callers). ++ if (KQ_rowsum_j != 0.0f) { ++ dstk_val.x /= KQ_rowsum_j; ++ dstk_val.y /= KQ_rowsum_j; ++ } else { ++ dstk_val.x = 0.0f; ++ dstk_val.y = 0.0f; ++ } + } + + if (is_fixup) { +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +index 26cd885..e2cde24 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -1001,7 +1001,13 @@ static __global__ void flash_attn_stream_k_fixup_uniform( + } + + // Write back final result: +- *dst = dst_val / rowsum; ++ // opencoti-hook: WS2 asym-band NaN guard (bug-2141) — the DCA fused kernel uses this shared stream-k ++ // fixup to combine its per-block partials; a fully-masked DCA band-boundary phase gives a zero exp-sum ++ // (rowsum==0), so the plain divide emits 0/0=NaN which the graph wht_o (dca.cpp) then butterfly-smears ++ // across all de-rotated output dims → NaN logits → empty generation (asym turbo_tcq K!=V @256k). Emit ++ // the identity 0 for a fully-masked row. NO-OP for rowsum!=0: the exact same division runs → ++ // bit-identical for every non-degenerate row (all FA callers of this fixup). ++ *dst = rowsum != 0.0f ? dst_val / rowsum : 0.0f; + + // opencoti-hook: rolling-kv M7 — per-row log-sum-exp for windowed-FA online-softmax combine (Stage 3a). See docs/protocols/UPSTREAM_SYNC.md. + if (dst_lse && tid == 0) { +@@ -1117,7 +1123,12 @@ static __global__ void flash_attn_stream_k_fixup_general( + } + + // Write back final result: +- *dst = dst_val / rowsum; ++ // opencoti-hook: WS2 asym-band NaN guard (bug-2141) — general (non-uniform block) variant of the ++ // shared stream-k fixup the DCA fused kernel combines through; a fully-masked DCA band-boundary phase ++ // gives a zero exp-sum (rowsum==0) → 0/0=NaN which the graph wht_o (dca.cpp) smears → empty generation ++ // (asym turbo_tcq K!=V @256k). Emit the identity 0. NO-OP for rowsum!=0: identical division → ++ // bit-identical for every non-degenerate row. ++ *dst = rowsum != 0.0f ? dst_val / rowsum : 0.0f; + + // opencoti-hook: rolling-kv M7 — per-row log-sum-exp for windowed-FA online-softmax combine (Stage 3a). See docs/protocols/UPSTREAM_SYNC.md. + if (dst_lse && tid == 0) { diff --git a/patches/0111-dca-streaming-tail-bug2144.patch b/patches/0111-dca-streaming-tail-bug2144.patch new file mode 100644 index 0000000000000000000000000000000000000000..6543575d542cc0b1e6a04aca46fd45f27d35b49a --- /dev/null +++ b/patches/0111-dca-streaming-tail-bug2144.patch @@ -0,0 +1,186 @@ +opencoti patch 0111 — DCA streaming-tail (bug-2144 / #635 / #631) + +Rolling-KV spill under DCA-on decode CLIFFED: DCA's build_attn_dca_core fetched +K/V via the whole-cache get_k/get_v accessors, which for a spilling POSITION_WINDOW +layer return the correctness fallback ggml_concat(window_device, tail_host). The +scheduler materializes that concat on the compute backend every decode step — a +synchronous, non-overlapped ~full-KV host<->device round-trip — so DCA never entered +the async tile streaming path the non-DCA build_attn_mha_position_window already uses. +Result: DCA-on decode held resident speed to an ~800 MiB tail then collapsed ~55x +(e.g. 113 -> 2.9 tps past ~1 GiB of tail), while DCA-off spilled gracefully. + +FIX (C1+C2): give the fused DCA kernel dca_fused its own two-region window+tail +streaming path, mirroring build_attn_mha_position_window. A spilling POSITION_WINDOW +layer reads get_k_window/get_v_window (device) + get_k_tail/get_v_tail (pinned host), +permutes the host tail into the same FA layout, and passes it to the fused kernel on +src[7]/src[8] with op_params[5]=n_win; the kernel walks window then tail under the +same DCA online-softmax. Gated by dca_window_decode: cparams.flash_attn AND small n_q +(decode / small spec-verify, WS2_NQ_MAX default 16) AND !single_chunk AND +!get_is_fragmented() AND POSITION_WINDOW tactic active AND headinfer_window_active — +so a fragmented multi-session / non-window cache can never enter it and the legacy +concat path is BYTE-IDENTICAL when off. + +DEFAULT-ON (WS2_DCA_STREAM=1 -> flip: escape WS2_DCA_STREAM=0 to force legacy concat). +Safe to default-on because the master flag only arms the path; dca_window_decode +enforces every real precondition structurally. + +VALIDATED (#637, C2ship binary, Qwen2.5-14B-1M Q8_0 @128k, bs2 RTX 6000, single-GPU): +DCA-on streaming spill tracks-AND-beats the DCA-off rolling-KV reference at every +matched tail — plateau ~37 tps to a ~4.9 GiB tail, common knee ~5.6 GiB (tail crossing +~1 DCA chunk = 32768 cells), graceful rolloff, NO cliff; needle PRESENT every cell. +~17x recovery at a 6.7 GiB tail (was ~0.68 -> 11.69 tps). Default-on re-gate (no env) +reproduces the streaming numbers. Marker: opencoti-hook: DCA streaming-tail +(bug-2144-C / Stage C) (dca.cpp, 2 sites). Host-only — no DSO change (the fused +kernel's src[7]/src[8] tail read is already in the shipped C2 DSO). Provenance: +opencoti-original. See docs/evaluations/rolling_kv.md §5d. Applies after 0110. + +diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp +index 4a314e382..efa73317b 100644 +--- a/llama.cpp/src/dca.cpp ++++ b/llama.cpp/src/dca.cpp +@@ -248,8 +248,48 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // the kernel). At f16 this is a no-op (byte-identical to the prior assert path). Context-shift + // stays correct independently: the modular K-shift re-ropes the STORED, still-quantized cache + // via build_rope_shift's existing dequant->rope->requant quantized branch (llama-kv-cache.cpp). +- ggml_tensor * k_raw = mctx_cur->get_k(ctx0, il); +- ggml_tensor * v_raw = mctx_cur->get_v(ctx0, il); ++ // opencoti-hook: DCA streaming-tail (bug-2144-C / Stage C). A spilling POSITION_WINDOW layer's get_k ++ // returns ggml_concat(window_device, tail_host); the scheduler round-trips the WHOLE KV host<->device ++ // every decode step (bug-2144: 38.4 -> 0.68 tps cliff the moment the tail crosses one DCA chunk). For ++ // DECODE (small n_q) on such a layer, read the device WINDOW + host TAIL via the split accessors and ++ // stream the tail inside the fused kernel (C2) instead of the concat. DEFAULT-ON (bug-2144 shipped: ++ // DCA-on spill tracks/beats the DCA-off rolling-KV reference at every matched tail, no cliff, needle ++ // PRESENT every cell — docs/evaluations/rolling_kv.md §5d); set WS2_DCA_STREAM=0 to force the legacy ++ // get_k/get_v whole-cache concat path (BYTE-IDENTICAL). SAFE to default-on: the master flag only ++ // arms the path; the dca_window_decode predicate below enforces every real precondition structurally ++ // (!get_is_fragmented, POSITION_WINDOW tactic active, headinfer_window_active, FA, decode n_q, ++ // multichunk), so a fragmented multi-session / non-window cache can never enter it. Requires the ++ // append-only (non-fragmented) analytical cache so band positions stay index==pos and src[7]/[8] are free. ++ static const bool ws2_dca_stream = []() { const char * e = getenv("WS2_DCA_STREAM"); return (!e || e[0] != '0'); }(); ++ static const int ws2_stream_nq = []() { const char * e = getenv("WS2_NQ_MAX"); return e ? atoi(e) : 16; }(); ++ ggml_tensor * k_stream_tail = nullptr; ++ ggml_tensor * v_stream_tail = nullptr; ++ int dca_n_win = 0; ++ const bool dca_window_decode = ws2_dca_stream ++ && cparams.flash_attn ++ && q_cur->ne[2] <= ws2_stream_nq ++ && !inp_dca->single_chunk ++ && !mctx_cur->get_is_fragmented() ++ && mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::POSITION_WINDOW ++ && mctx_cur->headinfer_window_active(il); ++ ggml_tensor * k_raw; ++ ggml_tensor * v_raw; ++ if (dca_window_decode) { ++ k_stream_tail = mctx_cur->get_k_tail(ctx0, il); // nullptr when the window covers the whole cache ++ v_stream_tail = mctx_cur->get_v_tail(ctx0, il); ++ if (k_stream_tail) { ++ k_raw = mctx_cur->get_k_window(ctx0, il); ++ v_raw = mctx_cur->get_v_window(ctx0, il); ++ dca_n_win = (int) k_raw->ne[2]; // window position-axis length (get_k_window: [dh, n_head_kv, n_win, 1]) ++ } else { ++ // window == whole cache (no overflow) → single-region read, byte-identical to get_k. ++ k_raw = mctx_cur->get_k(ctx0, il); ++ v_raw = mctx_cur->get_v(ctx0, il); ++ } ++ } else { ++ k_raw = mctx_cur->get_k(ctx0, il); ++ v_raw = mctx_cur->get_v(ctx0, il); ++ } + + // opencoti F5 dca (DCA all-KV C-perf, bug-720): on the analytical-band (multi-chunk) path the fused + // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see +@@ -375,12 +415,19 @@ ggml_tensor * llm_graph_context::build_attn_dca_core( + // opencoti F5 dca (#617B): get_mask(SUCC)/get_mask(INTER) are nullptr on the contiguous fast + // path (analytical bands) and the dense position-based F16 masks when the cache is fragmented; + // when non-null the kernel switches each phase to full-range + dense-mask selection. ++ // opencoti-hook: DCA streaming-tail (bug-2144-C / Stage C). Permute the host tail K/V into the ++ // same FA layout as kp/vp. C1: k_stream_tail/v_stream_tail are NULL (WS2_DCA_STREAM off) so both ++ // stay NULL and n_win==0 ⇒ the op is byte-identical to the single-region call. C2 first cut is ++ // scalar/f16 K/V (turbo-V stays on the f16-lift path), so a plain permute matches vp's treatment. ++ ggml_tensor * kp_tail = k_stream_tail ? ggml_permute(ctx0, k_stream_tail, 0, 2, 1, 3) : nullptr; ++ ggml_tensor * vp_tail = v_stream_tail ? ggml_permute(ctx0, v_stream_tail, 0, 2, 1, 3) : nullptr; + ggml_tensor * P = ggml_flash_attn_ext_dca_fused(ctx0, + qp[DCA_REGIME_INTRA], qp[DCA_REGIME_SUCC], qp[DCA_REGIME_INTER], + kp, vp, inp_dca->get_mask(DCA_REGIME_INTRA), + inp_dca->get_mask(DCA_REGIME_SUCC), inp_dca->get_mask(DCA_REGIME_INTER), + inp_dca->get_pos_q_real(), +- kq_scale, max_bias, softcap, (int) inp_dca->chunk_size); ++ kq_scale, max_bias, softcap, (int) inp_dca->chunk_size, ++ kp_tail, vp_tail, dca_n_win); + // dst is [DV+1, H, NQ, 1] (dense O block then UNUSED lse row); slice the dense O block. + const int64_t DV = P->ne[0] - 1; + const int64_t H = P->ne[1]; +diff --git a/llama.cpp/ggml/include/ggml.h b/llama.cpp/ggml/include/ggml.h +index 614da7f..00e5940 100644 +--- a/llama.cpp/ggml/include/ggml.h ++++ b/llama.cpp/ggml/include/ggml.h +@@ -2560,7 +2560,15 @@ extern "C" { + float scale, + float max_bias, + float logit_softcap, +- int chunk_size); ++ int chunk_size, ++ // opencoti-hook: DCA streaming-tail (bug-2144-C / Stage C). Two-region decode read: ++ // k/v carry the device-resident WINDOW [0,n_win); k_tail/v_tail carry the pinned-host ++ // overflow TAIL [n_win,n_kv). n_win==0 (both tail NULL) ⇒ legacy single-region fused op, ++ // byte-identical. Window mode is APPEND-ONLY (bug-268 swa_type==NONE) ⇒ non-fragmented ⇒ ++ // mask_succ/mask_inter are guaranteed NULL here, so the tail tensors reuse src[7]/src[8]. ++ struct ggml_tensor * k_tail, ++ struct ggml_tensor * v_tail, ++ int n_win); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, +diff --git a/llama.cpp/ggml/src/ggml.c b/llama.cpp/ggml/src/ggml.c +index 261d69e..69167f3 100644 +--- a/llama.cpp/ggml/src/ggml.c ++++ b/llama.cpp/ggml/src/ggml.c +@@ -5744,7 +5744,10 @@ struct ggml_tensor * ggml_flash_attn_ext_dca_fused( + float scale, + float max_bias, + float logit_softcap, +- int chunk_size) { ++ int chunk_size, ++ struct ggml_tensor * k_tail, ++ struct ggml_tensor * v_tail, ++ int n_win) { + GGML_ASSERT(ggml_can_mul_mat(k, q_intra)); + GGML_ASSERT(q_intra->ne[3] == k->ne[3]); + GGML_ASSERT(q_intra->ne[3] == v->ne[3]); +@@ -5774,10 +5777,23 @@ struct ggml_tensor * ggml_flash_attn_ext_dca_fused( + int64_t ne[4] = { v->ne[0] + 1, q_intra->ne[2], q_intra->ne[1], q_intra->ne[3] }; + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + ++ // opencoti-hook: DCA streaming-tail (bug-2144-C / Stage C). Two-region decode read. n_win>0 marks the ++ // device-window[0,n_win)+host-tail[n_win,n_kv) split; the tail tensors ride src[7]/src[8] (guaranteed ++ // free — window mode is append-only/non-fragmented so mask_succ/mask_inter are NULL). n_win==0 ⇒ legacy. ++ const bool dca_stream = (n_win > 0); ++ if (dca_stream) { ++ GGML_ASSERT(k_tail && v_tail && "dca stream: n_win>0 requires both tail tensors"); ++ GGML_ASSERT(mask_succ == NULL && mask_inter == NULL && ++ "dca stream: two-region path requires the append-only (non-fragmented) analytical bands"); ++ GGML_ASSERT(k_tail->type == k->type && v_tail->type == v->type); ++ GGML_ASSERT(k_tail->ne[0] == k->ne[0] && v_tail->ne[0] == v->ne[0]); ++ } ++ + float params[] = { scale, max_bias, logit_softcap }; + ggml_set_op_params(result, params, sizeof(params)); + ggml_set_op_params_i32(result, 3, chunk_size); + ggml_set_op_params_i32(result, 4, 3); // regime==3 ⇒ FUSED (CUDA forward branches on this) ++ ggml_set_op_params_i32(result, 5, n_win); // opencoti bug-2144-C: device-window length (0 ⇒ single-region) + + result->op = GGML_OP_FLASH_ATTN_EXT_LSE; + result->src[0] = q_intra; +@@ -5787,8 +5803,10 @@ struct ggml_tensor * ggml_flash_attn_ext_dca_fused( + result->src[4] = pos_q_real; // device I32 real abs query positions (band channel) + result->src[5] = q_succ; // pre-roped Q for the SUCC phase + result->src[6] = q_inter; // pre-roped Q for the INTER phase +- result->src[7] = mask_succ; // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ analytical fast path) +- result->src[8] = mask_inter; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ analytical fast path) ++ // src[7]/src[8]: dense SUCC/INTER masks on the fragmented fallback (NULL on the analytical fast path), ++ // OR — when n_win>0 (bug-2144-C two-region decode, always non-fragmented) — the host K/V overflow tail. ++ result->src[7] = dca_stream ? k_tail : mask_succ; ++ result->src[8] = dca_stream ? v_tail : mask_inter; + + return result; + } diff --git a/patches/0112-rolling-kv-phasegate-prefill-bug2145.patch b/patches/0112-rolling-kv-phasegate-prefill-bug2145.patch new file mode 100644 index 0000000000000000000000000000000000000000..e4462ff3553712642df3916f75451ca52742b6c5 --- /dev/null +++ b/patches/0112-rolling-kv-phasegate-prefill-bug2145.patch @@ -0,0 +1,79 @@ +opencoti patch 0112 — rolling-KV phase-gate prefill (bug-2145 / #636 / #631) + +The non-DCA streaming-window fast-path (build_attn_mha_position_window) has NO +batched-MMA large-n_q kernel, so PREFILL ran decode-style over the whole resident +window -> a flat, tail-INDEPENDENT ~325 tok/s prefill cliff whenever the rolling-KV +window engaged (any spill config on a full-attention model). + +FIX: gate the streaming-window path to DECODE / small spec-verify (n_q <= WS2_NQ_MAX, +default 16, mirroring DCA's dca.cpp gate). Large-n_q PREFILL falls to an explicit +else branch (path B) that reads the whole-cache concat accessor get_k/get_v and FAs +it in one amortized batched build_attn_mha (ggml_flash_attn_ext) pass — the same +machinery that already makes DCA prefill graceful, whose concat H2D round-trip +amortizes over the microbatch (tail-dependent / graceful instead of the flat cliff). +Applied at both window-mode dispatch sites (full-attention overload + the second +build_attn path). The explicit else also keeps a window layer from leaking into the +GPU_STREAM/neo branch below. + +VALIDATED (#637): the DCA-off rolling-KV ladder prefill holds ~1340 tok/s on the +plateau (path B amortized) instead of the ~325 flat cliff, decode graceful, needle +PRESENT every cell. Marker: opencoti-hook: rolling-KV phase-gate prefill (bug-2145) +(llama-graph.cpp, 3 sites). Host-only — no DSO change. Provenance: opencoti-original. +See docs/evaluations/rolling_kv.md §5c/§5d. Applies after 0111. + +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index a709dbe7f..874fa9017 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2996,7 +2996,18 @@ ggml_tensor * llm_graph_context::build_attn( + cparams.flash_attn && kq_b == nullptr && v_mla == nullptr && sinks == nullptr && + mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::POSITION_WINDOW && + mctx_cur->headinfer_window_active(il); +- if (window_layer) { ++ // opencoti-hook: rolling-KV phase-gate prefill (bug-2145) — the streaming-window fast-path ++ // build_attn_mha_position_window has NO batched-MMA large-n_q kernel, so PREFILL runs ++ // decode-style over the whole window → a flat, tail-INDEPENDENT ~325 tok/s cliff (bug-2145). ++ // Gate path A (streaming window) to DECODE / small spec-verify (small n_q); route large-n_q ++ // PREFILL through the whole-cache concat accessor get_k/get_v + batched build_attn_mha ++ // (ggml_flash_attn_ext) — path B, the same machinery that already makes DCA prefill graceful, ++ // whose concat H2D round-trip amortizes over the microbatch. Mirrors DCA's WS2_NQ_MAX ++ // (dca.cpp:300). q->ne[2] is the per-ubatch n_q. See bug-2145, docs/evaluations/rolling_kv.md §5c. ++ static const int64_t ws2_window_nq_max = []() { ++ const char * e = getenv("WS2_NQ_MAX"); return (int64_t) (e ? atoi(e) : 16); ++ }(); ++ if (window_layer && q->ne[2] <= ws2_window_nq_max) { + ggml_tensor * k_win = mctx_cur->get_k_window(ctx0, il); + ggml_tensor * v_win = mctx_cur->get_v_window(ctx0, il); + ggml_tensor * k_tail = mctx_cur->get_k_tail (ctx0, il); // nullptr when n_kv <= wc +@@ -3005,6 +3016,14 @@ ggml_tensor * llm_graph_context::build_attn( + (k_tail != nullptr) && (q->ne[2] == 1) && mctx_cur->headinfer_tail_is_cpu_fa(); + cur = build_attn_mha_position_window(q, k_win, v_win, k_tail, v_tail, + kq_b, kq_mask, sinks, v_mla, kq_scale, il, cpu_fa_tail); ++ } else if (window_layer) { ++ // opencoti-hook: rolling-KV phase-gate prefill (bug-2145) — path B: large-n_q prefill. ++ // get_k/get_v return ggml_concat(window_device, tail_host); batched build_attn_mha FAs ++ // it in one amortized pass (tail-dependent/graceful) instead of the decode-style window ++ // cliff. Explicit branch so a window layer never leaks into GPU_STREAM/neo below. ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il); ++ ggml_tensor * v = mctx_cur->get_v(ctx0, il); ++ cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); + } else if (mctx_cur->get_layer_tactic(il) == llama_kv_cache::headinfer_tactic::GPU_STREAM) { + // opencoti F5 M7 Rolling KV — Rung 2 (W4 M7-D #304), S1. GPU_STREAM + // rides the M2 head-split: the device head-group runs a resident FA, +@@ -3322,7 +3341,15 @@ ggml_tensor * llm_graph_context::build_attn( + mctx_cur->headinfer_window_active(il); + + ggml_tensor * cur; +- if (window_layer) { ++ // opencoti-hook: rolling-KV phase-gate prefill (bug-2145) — gate the streaming-window ++ // fast-path to DECODE / small spec-verify; large-n_q PREFILL falls to the else branch ++ // (get_k/get_v concat + batched build_attn_mha = ggml_flash_attn_ext), avoiding the flat ++ // tail-independent prefill cliff. Mirrors the full-attention overload above and DCA's ++ // WS2_NQ_MAX (dca.cpp:300). See bug-2145, docs/evaluations/rolling_kv.md §5c. ++ static const int64_t ws2_window_nq_max = []() { ++ const char * e = getenv("WS2_NQ_MAX"); return (int64_t) (e ? atoi(e) : 16); ++ }(); ++ if (window_layer && q->ne[2] <= ws2_window_nq_max) { + ggml_tensor * k_win = mctx_cur->get_k_window(ctx0, il); + ggml_tensor * v_win = mctx_cur->get_v_window(ctx0, il); + ggml_tensor * k_tail = mctx_cur->get_k_tail (ctx0, il); // nullptr when n_kv <= wc diff --git a/patches/0113-rolling-kv-reserve-fixpoint-bug2146.patch b/patches/0113-rolling-kv-reserve-fixpoint-bug2146.patch new file mode 100644 index 0000000000000000000000000000000000000000..c40dcf529736112ca438c43e75a05ebf2a752ade --- /dev/null +++ b/patches/0113-rolling-kv-reserve-fixpoint-bug2146.patch @@ -0,0 +1,118 @@ +opencoti patch 0113 — rolling-KV reserve fixpoint (bug-2146 / #637 / #631) + +The bug-1342 two-pass compute-reserve sizer measured the GPU compute buffer in pass 1 +against a DEGENERATE minimal resident window (wc=256, kv_window_measure_pass) so +sched_reserve cannot OOM. With the bug-2144 streaming dca_fused decode path active, +that tiny window forces a MAXIMAL host tail, and the streaming op's tail-tile scratch +scales with the tail -> the measured buffer inflated to ~4088 MiB vs the real ~926 MiB, +a ~3.16 GiB over-reservation. Consequence: a KV that fits in VRAM was pushed to a +~3168 MiB host tail (37.1 tps) instead of staying fully resident (~40). + +FIX: iterate the two-pass reserve to a FIXPOINT (llama-context.cpp, cap 3 iterations, +128 MiB convergence margin). Each re-size grows the resident window -> shrinks the tail +-> shrinks the measured buffer, so the reserve decreases monotonically and converges to +the plain resident buffer (tail 0) when the KV fits. The resident graph's buffer is a +subset of the streaming graph's, so a re-size only ever reveals a smaller-or-equal +buffer -> it can never under-reserve. For the non-streaming case pass 2 measures what +pass 1 did -> converges in ONE step, BYTE-IDENTICAL to the prior single-shot behaviour. + +VALIDATED (#637, C2r/C2ship binary, Qwen2.5-14B-1M Q8_0 @128k, bs2): vt=41888 (KV +fits) 3168 MiB tail -> FULLY RESIDENT, 37.10 -> 38.40 tps, converges in 2 iters, needle +PRESENT; vt=35400 (genuine spill) tail 9648 -> 6720 MiB, 7.24 -> 11.72 tps (+62%), +needle PRESENT. Recovers ~3.16 GiB and grows the resident window at every vram-target. +Host-only — no DSO change. Provenance: opencoti-original. See +docs/evaluations/rolling_kv.md §5d. Applies after 0112. + +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 1579acc88..e40aa9e27 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -478,34 +478,63 @@ llama_context::llama_context( + // re-create the KV with that measured reserve, and re-reserve. The compute buffer is + // independent of the window/tail split, so the measurement is exact for pass 2. See + // docs/features/rolling_kv_compute_reserve.md. ++ // opencoti bug-2146: iterate the reserve to a fixpoint. Pass 1 measures the compute ++ // buffer against a DEGENERATE minimal window (wc=256, kv-cache.cpp measure_pass): ++ // when the streaming dca_fused / window-tail decode path is active that tiny window ++ // forces a MAXIMAL host tail, and the streaming op's tail-tile scratch scales with ++ // the tail — so the measured buffer is inflated (observed 4088 MiB vs the real 926 ++ // MiB at a realistic window). Sizing the window against that inflated reserve forces ++ // a needlessly large host tail (a ~3 GiB spill even when the KV would fit resident). ++ // The comment's old claim "compute buffer is independent of the window/tail split" ++ // holds for the plain resident graph but NOT for the streaming path. Fix: re-size, ++ // re-measure, and repeat — each re-size grows the window → shrinks the tail → shrinks ++ // the measured buffer, so the reserve decreases monotonically and converges (to the ++ // plain resident buffer, tail 0, when the KV fits). For the non-streaming case pass 2 ++ // measures the same value pass 1 did, so we converge in ONE step (identical to the ++ // prior behaviour). Capped at a few iterations; stops as soon as the reserve stops ++ // shrinking. See docs/evaluations/rolling_kv.md §5d. + if (opencoti_two_pass && !model.hparams.no_alloc) { +- size_t gpu_compute = 0; +- for (size_t i = 0; i < backend_ptrs.size(); ++i) { +- if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) == GGML_BACKEND_DEVICE_TYPE_GPU) { +- gpu_compute += backend_buf_exp_size[i]; ++ uint32_t prev_reserve_mib = UINT32_MAX; ++ for (int reserve_iter = 0; reserve_iter < 3; ++reserve_iter) { ++ size_t gpu_compute = 0; ++ for (size_t i = 0; i < backend_ptrs.size(); ++i) { ++ if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) == GGML_BACKEND_DEVICE_TYPE_GPU) { ++ gpu_compute += backend_buf_exp_size[i]; ++ } + } +- } +- // + 256 MiB safety margin over the measured worst-case pp compute buffer. +- const uint32_t measured_mib = (uint32_t) (gpu_compute / (1024 * 1024)) + 256; +- LLAMA_LOG_INFO("%s: rolling-kv two-pass — measured GPU compute buffer = %u MiB; " +- "re-sizing resident window against it\n", __func__, measured_mib); +- cparams.kv_compute_reserve_mib = measured_mib; +- cparams.kv_window_measure_pass = false; +- +- // free the pass-1 KV (incl. its host tail) BEFORE creating pass 2 to avoid a +- // transient double host-tail allocation. +- memory.reset(); +- llama_memory_params params_mem2 = { +- /*.type_k =*/ params.type_k, +- /*.type_v =*/ params.type_v, +- /*.swa_full =*/ params.swa_full, +- /*.ctx_type= */ cparams.ctx_type, +- /*.mem_other=*/ cparams.ctx_other ? llama_get_memory(cparams.ctx_other) : nullptr, // opencoti bug-858 +- }; +- memory.reset(model.create_memory(params_mem2, cparams)); ++ // + 256 MiB safety margin over the measured worst-case pp compute buffer. ++ const uint32_t measured_mib = (uint32_t) (gpu_compute / (1024 * 1024)) + 256; ++ ++ // Converged: the freshly measured reserve is no meaningfully smaller than the ++ // one the window is already sized against (128 MiB margin) → another re-size ++ // cannot free enough VRAM to matter. Never GROWS the reserve here: the resident ++ // graph's buffer is a subset of the streaming graph's, so re-sizing only ever ++ // reveals a smaller-or-equal buffer, and a break can never under-reserve. ++ if (measured_mib + 128 >= prev_reserve_mib) { ++ break; ++ } ++ LLAMA_LOG_INFO("%s: rolling-kv two-pass — measured GPU compute buffer = %u MiB; " ++ "re-sizing resident window against it (iter %d)\n", ++ __func__, measured_mib, reserve_iter); ++ prev_reserve_mib = measured_mib; ++ cparams.kv_compute_reserve_mib = measured_mib; ++ cparams.kv_window_measure_pass = false; ++ ++ // free the current KV (incl. its host tail) BEFORE creating the next to avoid a ++ // transient double host-tail allocation. ++ memory.reset(); ++ llama_memory_params params_mem2 = { ++ /*.type_k =*/ params.type_k, ++ /*.type_v =*/ params.type_v, ++ /*.swa_full =*/ params.swa_full, ++ /*.ctx_type= */ cparams.ctx_type, ++ /*.mem_other=*/ cparams.ctx_other ? llama_get_memory(cparams.ctx_other) : nullptr, // opencoti bug-858 ++ }; ++ memory.reset(model.create_memory(params_mem2, cparams)); + +- sched_need_reserve = true; +- sched_reserve(); ++ sched_need_reserve = true; ++ sched_reserve(); ++ } + } + + if (!cparams.flash_attn) { diff --git a/patches/0114-hybrid-rolling-kv-residency-bug2142.patch b/patches/0114-hybrid-rolling-kv-residency-bug2142.patch new file mode 100644 index 0000000000000000000000000000000000000000..b3b5c92ac66491f4aef9ebf3be68f58af72c23f8 --- /dev/null +++ b/patches/0114-hybrid-rolling-kv-residency-bug2142.patch @@ -0,0 +1,189 @@ +opencoti patch 0114 — rolling-KV residency for hybrid memory wrappers (bug-2142 / #633) + +Rolling-KV residency knobs (--vram-target, --kv-residency-mode, the bug-1342 +compute-reserve, and the window-measure-pass) reached the iSWA (llama-model.cpp +llama_kv_cache_iswa) and plain-dense (llama_kv_cache) memory paths but were SILENTLY +DROPPED for hybrid models: llama_memory_hybrid and llama_memory_hybrid_iswa hardcoded +0/0.0f/false when constructing their internal attention llama_kv_cache, so a hybrid +model (qw35a3 = Qwen3.6-35B-A3B DeltaNet+attn NextN MoE) on a too-small card IGNORED +--vram-target and OOMed instead of doing graceful rolling-KV overflow. + +FIX (additive, host-only, no CUDA DSO change): thread cparams.vram_target_mib, +pcie_bw_gbps, kv_residency_mode, kv_compute_reserve_mib and kv_window_measure_pass +through both hybrid wrapper ctors down to the attention cache, mirroring the iSWA/plain +forwarding. The new ctor params default to the prior hardcoded off-state, so any caller +that does not forward them (and every resident/no---vram-target config) stays +byte-identical. The slot_initial_ctx / slot_shrink_idle_ms / headinfer knobs remain +hardcoded off for hybrid (separate F4-M3 / F5-M2 features, out of scope here). + +Regression shield: non-hybrid (iSWA Gemma, dense 14B/31B) residency untouched; a +resident hybrid config is byte-identical; only a hybrid + --vram-target-below-KV boot +changes behaviour (now: engages POSITION_WINDOW / CPU_SPILL with a host tail, VT-derived +budget, instead of OOM). + +git apply ignores this prose preamble and starts at the first "diff --git" below. + +diff --git a/llama.cpp/src/llama-memory-hybrid.h b/llama.cpp/src/llama-memory-hybrid.h +index 484eafb..b7540c5 100644 +--- a/llama.cpp/src/llama-memory-hybrid.h ++++ b/llama.cpp/src/llama-memory-hybrid.h +@@ -37,6 +37,16 @@ public: + uint32_t n_rs_seq, + bool offload, + bool unified, ++ /* opencoti F5 M7 rolling-KV residency (bug-2142) — ++ forwarded to the inner attention llama_kv_cache so ++ hybrid models honour --vram-target et al. instead of ++ silently OOMing; defaults reproduce the prior ++ hardcoded off-state for any non-forwarding caller. */ ++ uint32_t vram_target_mib = 0, ++ float pcie_bw_gbps = 0.0f, ++ uint32_t kv_residency_mode = 0, ++ uint32_t kv_compute_reserve_mib = 0, ++ bool kv_window_measure_pass = false, + /* layer filters */ + const layer_filter_cb & filter_attn = nullptr, + const layer_filter_cb & filter_recr = nullptr); +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +index 0fa114d..6c22724 100644 +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -27,6 +27,12 @@ llama_memory_hybrid::llama_memory_hybrid( + uint32_t n_rs_seq, + bool offload, + bool unified, ++ /* opencoti F5 M7 rolling-KV residency (bug-2142) */ ++ uint32_t vram_target_mib, ++ float pcie_bw_gbps, ++ uint32_t kv_residency_mode, ++ uint32_t kv_compute_reserve_mib, ++ bool kv_window_measure_pass, + /* layer filters */ + const layer_filter_cb & filter_attn, + const layer_filter_cb & filter_recr) : +@@ -49,15 +55,16 @@ llama_memory_hybrid::llama_memory_hybrid( + 0, + // opencoti F5 M2 headinfer — hybrid memory does not thread headinfer; 1.0 = off. + 1.0f, +- // opencoti F5 M7 Rolling KV — hybrid memory does not thread auto residency; 0 = unused. +- 0, +- // opencoti F5 M7 Stage 3d-S2 — hybrid memory does not thread PCIe bandwidth; 0 = unknown (stream default). +- 0.0f, +- // opencoti F5 M7 Stage 4 — hybrid memory does not thread the residency-mode knob; 0 = auto. +- 0, +- // opencoti F5 M7 bug-1342 — hybrid memory does not use the window two-pass; 0 / false. +- 0, +- false, ++ // opencoti F5 M7 Rolling KV (bug-2142) — thread --vram-target so hybrid models ++ // honour the residency cap (graceful spill) instead of silently OOMing. ++ vram_target_mib, ++ // opencoti F5 M7 Stage 3d-S2 (bug-2142) — thread PCIe bandwidth for the tactic selector. ++ pcie_bw_gbps, ++ // opencoti F5 M7 Stage 4 (bug-2142) — thread the residency-mode knob. ++ kv_residency_mode, ++ // opencoti F5 M7 bug-1342 (bug-2142) — thread the window two-pass compute reserve. ++ kv_compute_reserve_mib, ++ kv_window_measure_pass, + n_seq_max, + n_pad, + n_swa, +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.h b/llama.cpp/src/llama-memory-hybrid-iswa.h +index c9d3f9f..2966688 100644 +--- a/llama.cpp/src/llama-memory-hybrid-iswa.h ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.h +@@ -37,6 +37,16 @@ public: + uint32_t n_rs_seq, + bool offload, + bool unified, ++ /* opencoti F5 M7 rolling-KV residency (bug-2142) — ++ forwarded to the inner attention llama_kv_cache_iswa so ++ hybrid-iswa models honour --vram-target et al. instead of ++ silently OOMing; defaults reproduce the prior hardcoded ++ off-state for any non-forwarding caller. */ ++ uint32_t vram_target_mib = 0, ++ float pcie_bw_gbps = 0.0f, ++ uint32_t kv_residency_mode = 0, ++ uint32_t kv_compute_reserve_mib = 0, ++ bool kv_window_measure_pass = false, + /* layer filters */ + const layer_filter_cb & filter_attn = nullptr, + const layer_filter_cb & filter_recr = nullptr); +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +index 3cf8e4a..f618219 100644 +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -27,6 +27,12 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + uint32_t n_rs_seq, + bool offload, + bool unified, ++ /* opencoti F5 M7 rolling-KV residency (bug-2142) */ ++ uint32_t vram_target_mib, ++ float pcie_bw_gbps, ++ uint32_t kv_residency_mode, ++ uint32_t kv_compute_reserve_mib, ++ bool kv_window_measure_pass, + /* layer filters */ + const layer_filter_cb & filter_attn, + const layer_filter_cb & filter_recr) : +@@ -50,15 +56,16 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + 0, + // opencoti F5 M2 headinfer — hybrid-iswa does not thread headinfer; 1.0 = off. + 1.0f, +- // opencoti F5 M7 Rolling KV — hybrid-iswa does not thread auto residency; 0 = unused. +- 0, +- // opencoti F5 M7 Stage 3d-S2 — hybrid-iswa does not thread PCIe bandwidth; 0 = unknown (stream default). +- 0.0f, +- // opencoti F5 M7 Stage 4 — hybrid-iswa does not thread the residency-mode knob; 0 = auto. +- 0, +- // opencoti F5 M7 bug-1342 — hybrid-iswa does not use the window two-pass; 0 / false. +- 0, +- false, ++ // opencoti F5 M7 Rolling KV (bug-2142) — thread --vram-target so hybrid-iswa models ++ // honour the residency cap (graceful spill) instead of silently OOMing. ++ vram_target_mib, ++ // opencoti F5 M7 Stage 3d-S2 (bug-2142) — thread PCIe bandwidth for the tactic selector. ++ pcie_bw_gbps, ++ // opencoti F5 M7 Stage 4 (bug-2142) — thread the residency-mode knob. ++ kv_residency_mode, ++ // opencoti F5 M7 bug-1342 (bug-2142) — thread the window two-pass compute reserve. ++ kv_compute_reserve_mib, ++ kv_window_measure_pass, + n_seq_max, + n_ubatch, + n_pad, +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +index d7804be..b085d9d 100644 +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2035,6 +2035,15 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + /* n_rs_seq */ cparams.n_rs_seq, + /* offload */ cparams.offload_kqv, + /* unified */ cparams.kv_unified, ++ // opencoti-hook: rolling-KV residency for hybrid (bug-2142) — see ++ // docs/features/rolling_kv.md. Thread --vram-target et al. into the ++ // hybrid-iswa wrapper so DeltaNet+attn NextN models (qw35a3) get ++ // graceful spill instead of a silent OOM, mirroring the iSWA/dense paths. ++ /* vram_target_mib */ cparams.vram_target_mib, ++ /* pcie_bw_gbps */ cparams.pcie_bw_gbps, ++ /* kv_residency_mode */ cparams.kv_residency_mode, ++ /* kv_compute_reserve_mib */ cparams.kv_compute_reserve_mib, ++ /* kv_window_measure_pass */ cparams.kv_window_measure_pass, + /* filter_attn */ std::move(filter_attn), + /* filter_recr */ std::move(filter_recr)); + } else { +@@ -2054,6 +2063,15 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + /* n_rs_seq */ cparams.n_rs_seq, + /* offload */ cparams.offload_kqv, + /* unified */ cparams.kv_unified, ++ // opencoti-hook: rolling-KV residency for hybrid (bug-2142) — see ++ // docs/features/rolling_kv.md. Thread --vram-target et al. into the ++ // hybrid wrapper so DeltaNet+attn NextN models (qw35a3, no SWA) get ++ // graceful spill instead of a silent OOM, mirroring the iSWA/dense paths. ++ /* vram_target_mib */ cparams.vram_target_mib, ++ /* pcie_bw_gbps */ cparams.pcie_bw_gbps, ++ /* kv_residency_mode */ cparams.kv_residency_mode, ++ /* kv_compute_reserve_mib */ cparams.kv_compute_reserve_mib, ++ /* kv_window_measure_pass */ cparams.kv_window_measure_pass, + /* filter_attn */ std::move(filter_attn), + /* filter_recr */ std::move(filter_recr)); + } diff --git a/patches/0115-rolling-kv-shift-guard-bug2148.patch b/patches/0115-rolling-kv-shift-guard-bug2148.patch new file mode 100644 index 0000000000000000000000000000000000000000..9cae670247bbe8ef35d1e7a089cff1b839df872b --- /dev/null +++ b/patches/0115-rolling-kv-shift-guard-bug2148.patch @@ -0,0 +1,60 @@ +opencoti patch 0115 — refuse context-shift on a spilled rolling-KV window (bug-2148 / #638) + +The F5 M7 Stage-3c "position window" splits the KV cache across a GPU-resident window +[0,wc) and a CPU-resident tail [wc,kv_size) when --vram-target forces overflow. Context +shift (server seq_rm + seq_add → pos_add) and self-extend (seq_div → pos_div) re-rope the +cache IN PLACE via the k_shift graph. That graph views only k_per_stream — sized to the +window's wc cells, NOT get_size() — so the moment a shift fires with a spilled window it +over-reads the window tensor and hits ggml.c GGML_ASSERT(data_size + view_offs <= +ggml_nbytes(view_src)); and even if the view fit, the host tail (written already-roped, +never transformed again) would be left stale-roped. There is no in-place path to re-rope +the CPU tail. + +FIX (additive, host-only, no CUDA DSO change): llama_kv_cache::get_can_shift() now returns +false when any layer has an active spilled window (window_cells > 0 && a populated +k_cpu_per_stream). The server reads this at init (common_init_from_params / +server-context.cpp:1093) and cleanly disables ctx_shift + cache-reuse, so a spilled slot is +BOUNDED at n_ctx instead of crashing. Fully-resident windows (wc == kv_size, no tail, +window_cells sentinel 0) and every non-window cache find no spilled layer and are +byte-identical — the fully-resident path still context-shifts normally. iSWA and hybrid +wrappers already AND / delegate their inner caches' get_can_shift, so the leaf guard +propagates. The "no-degradation" alternative (a host-side rope-by-delta pass over the CPU +tail on every shift) is deferred as out of scope for the multi-session prefix-shared serving +target, which does not rely on infinite-context shift over a spilled tail. + +Gate (bs2, Qwen2.5-14B-Instruct-1M-Q8_0, -c 12288, --context-shift --keep 256): + windowON (--vram-target 18000, 6912 resident / 5376 host tail): shift disabled at init, + request bounded at n_ctx (predicted_n=3840, 8448+3840=12288), coherent, NO assert. + controlOFF(--vram-target 80000, fully resident): shift STILL fires (n_discard=6015), + coherent to 1219 — guard does not over-fire. +Prior binary asserted at ggml.c:1840 on the windowON shift. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2460,6 +2460,25 @@ + if (hparams.n_pos_per_embd() > 1) { + return false; + } ++ // opencoti-hook: rolling-kv-shift-guard — see docs/features/rolling_kv_step_prefetch.md (#638, bug-2148) ++ // A spilled position window (F5 M7 Stage 3c) stores the cache split across a GPU-resident ++ // window [0,wc) and a CPU-resident tail [wc,kv_size). Context-shift (seq_add→pos_add) and ++ // self-extend (seq_div→pos_div) re-rope the cache IN PLACE via the k_shift graph — but that ++ // graph views only k_per_stream (sized wc, NOT get_size()) and has no path to re-rope the host ++ // tail. Attempting a shift therefore (a) asserts in the k_shift view (ggml.c: data_size + ++ // view_offs <= nbytes, over-reading the wc-cell window as get_size() cells) and (b) would leave ++ // the tail stale-roped even if the view fit. Refuse the shift while a spilled window is active: ++ // server-context.cpp:1093 reads this at init and cleanly disables ctx_shift / cache-reuse, so the ++ // request is bounded at n_ctx instead of crashing. Fully-resident windows (wc==kv_size, no tail, ++ // window_cells sentinel 0) and non-window caches are UNAFFECTED (loop finds no spilled layer). ++ // The no-degradation alternative (re-rope the host tail on every shift) is deferred. ++ for (const auto & layer : layers) { ++ if (layer.window_cells > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[0] != nullptr) { ++ return false; ++ } ++ } + return true; + } + diff --git a/patches/0116-kv-auto-tier-hal582-p1.patch b/patches/0116-kv-auto-tier-hal582-p1.patch new file mode 100644 index 0000000000000000000000000000000000000000..669d301689e6ce2ca86f0ecea9af9b5ea3946dec --- /dev/null +++ b/patches/0116-kv-auto-tier-hal582-p1.patch @@ -0,0 +1,167 @@ +opencoti patch 0116 — auto KV-tier selection at boot (HAL #582 P1 / #621) + +The HAL #582 study (docs/evaluations/hal_kv_compression.md) settled that on a +dense D128 model (Qwen2.5-14B-1M) the right multi-session capacity lever is a +RESIDENT quantized KV cache — pick the LEAST-compressing scalar KV type pair +that still keeps the whole cache on-GPU, rather than spilling f16 to host. P0 +(#620) already made quantized-KV decode bandwidth-bound so the quant tiers cost +throughput, not correctness. P1 is the policy: choose that tier automatically at +boot instead of making the operator hand-tune -ctk/-ctv. + +MECHANISM (additive, host-only, no CUDA DSO change). A new static helper +`opencoti_auto_select_kv_tier` runs in the llama_kv_cache ctor BEFORE any +residency/allocation math consumes type_k/type_v. It reuses the EXACT budget +arithmetic the rolling-KV window sizer uses (free VRAM − already-resident − +compute-reserve, capped by --vram-target) and the same per-layer per-cell byte +sum, then walks a fixed ladder — f16 → q8_0/q8_0 → q8_0/q4_0 → q5_1/q4_0 (the +HAL §2 hi-K/cheap-V recipe; q5_1/q4_0 is the last-resort floor) — and selects +the first (least-compressing) rung whose whole-cache byte cost fits the budget. +If even the floor does not fit, the floor is chosen and the rolling-KV window +then spills the excess (HAL §5 P1: "spill only if even the floor doesn't fit"). +The chosen tier is announced at WARN with a "set -ctk/-ctv to override" hint, +mirroring the sparse-V auto-policy (#567). + +STRICTLY OPT-IN + BYTE-IDENTICAL OTHERWISE. The helper is a no-op unless +OPENCOTI_KV_AUTO_TIER is set in the environment. It also returns early — leaving +type_k/type_v untouched — when: there is no GPU offload; the user set -ctk/-ctv +to anything other than the f16 default (the override is respected); the model is +iSWA (swa_type != NONE, e.g. Gemma keeps its validated f16/resident class, HAL +§3c); or the top rung (f16) already fits. In every one of those cases, and in +every existing caller (env unset), type_k/type_v are unchanged, so the boot is +byte-identical to the shipped path. + +VALIDATION (bs2, Qwen2.5-14B-Instruct-1M Q8_0, -c 32768, RTX 6000). With +OPENCOTI_KV_AUTO_TIER=1 the ladder walks correctly as --vram-target tightens: +generous → f16 (no WARN, byte-identical); 20000 MiB → q8_0/q8_0 (KV 3264 MiB +fits budget 3328); 19400 → q8_0/q4_0 (2496 fits 2728); 18800 → q5_1/q4_0 (2016 +fits 2128). Every chosen tier logged "position window = FULLY RESIDENT … no host +tail" and returned the passkey needle cleanly. Config-equivalence: the auto +q8_0/q8_0 boot produced byte-identical output to an explicit `-ctk q8_0 -ctv +q8_0` boot, which itself emitted zero auto-tier WARN (opt-in path confirmed off). + +opencoti-hook: kv-auto-tier — registered in docs/protocols/UPSTREAM_SYNC.md. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -355,6 +355,106 @@ static uint32_t opencoti_compute_resident_window_cells( + return cells; + } + ++// opencoti-hook: kv-auto-tier — see docs/evaluations/hal_kv_compression.md §5 P1 (#621, HAL #582) ++// Auto KV-tier selection. When opted in (env OPENCOTI_KV_AUTO_TIER=1) and the user LEFT -ctk/-ctv ++// at the f16 default, pick the LEAST-compressing scalar KV type pair that keeps the WHOLE cache ++// fully resident within the SAME free-VRAM budget the rolling-KV window uses (free − already-resident ++// − compute-reserve, capped by --vram-target). This gives multi-session capacity without the user ++// hand-tuning -ctk/-ctv. If even the floor won't fit, the floor is chosen and the rolling-KV window ++// then spills the excess (HAL §5 P1: "spill only if even the floor doesn't fit"). Scoped to the ++// VALIDATED class: dense full-attention (swa_type==NONE, e.g. Qwen D128); iSWA (Gemma) keeps its f16 ++// default — its class is not on this scalar ladder (HAL §3c: A4B stays f16/resident anyway). If the ++// top rung (f16) already fits, type_k/type_v are UNCHANGED ⇒ byte-identical boot. Announced at WARN ++// with an override hint, mirroring the sparse-V auto-policy (#567). Mutates type_k/type_v in place. ++static void opencoti_auto_select_kv_tier( ++ const llama_model & model, const llama_hparams & hparams, ++ ggml_type & type_k, ggml_type & type_v, ++ bool v_trans, bool offload, uint32_t kv_size, uint32_t n_stream, ++ uint32_t vram_target_mib, llama_swa_type swa_type, ++ const std::function & filter) { ++ if (getenv("OPENCOTI_KV_AUTO_TIER") == nullptr) { ++ return; // opt-in only — silent + byte-identical when unset ++ } ++ if (!offload || kv_size == 0) { ++ return; // no GPU offload → no residency pressure to relieve ++ } ++ if (type_k != GGML_TYPE_F16 || type_v != GGML_TYPE_F16) { ++ return; // user set -ctk/-ctv explicitly → respect the override ++ } ++ if (swa_type != LLAMA_SWA_TYPE_NONE) { ++ return; // dense full-attention is the validated class; iSWA keeps f16 ++ } ++ ++ // Budget — identical math to opencoti_compute_resident_window_cells (type-independent). ++ ggml_backend_dev_t dev = nullptr; ++ for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ if (!hparams.has_kv(il) || (filter && !filter(il))) { ++ continue; ++ } ++ dev = model.dev_layer(il); ++ break; ++ } ++ if (!dev) { ++ return; ++ } ++ size_t free_vram = 0, total_vram = 0; ++ ggml_backend_dev_memory(dev, &free_vram, &total_vram); ++ size_t budget = free_vram; ++ if (vram_target_mib > 0) { ++ const size_t vt_bytes = (size_t) vram_target_mib * 1024 * 1024; ++ const size_t already_used = total_vram > free_vram ? total_vram - free_vram : 0; ++ const size_t vt_kv_share = vt_bytes > already_used ? vt_bytes - already_used : 0; ++ budget = std::min(budget, vt_kv_share); ++ } ++ budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES ++ : 0; ++ ++ // Per-cell KV bytes for a candidate (tk,tv) — same per-layer sum as the residency sizer. ++ auto per_cell_bytes = [&](ggml_type tk, ggml_type tv) -> size_t { ++ size_t pc = 0; ++ for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ if (!hparams.has_kv(il) || (filter && !filter(il))) { ++ continue; ++ } ++ const uint32_t nk = hparams.n_embd_k_gqa(il); ++ const uint32_t nv = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max(); ++ pc += (size_t) ggml_row_size(tk, nk); ++ pc += (size_t) ggml_row_size(tv, nv); ++ } ++ return pc * (size_t) n_stream; ++ }; ++ ++ // Ladder: least → most compression (HAL §5 P1 / §2 quality bar). q8_0/q4_0 is the ++ // recommended hi-K/cheap-V recipe (§2); q5_1/q4_0 is the last-resort floor. ++ struct kv_tier { ggml_type k, v; }; ++ static const kv_tier ladder[] = { ++ { GGML_TYPE_F16, GGML_TYPE_F16 }, ++ { GGML_TYPE_Q8_0, GGML_TYPE_Q8_0 }, ++ { GGML_TYPE_Q8_0, GGML_TYPE_Q4_0 }, ++ { GGML_TYPE_Q5_1, GGML_TYPE_Q4_0 }, ++ }; ++ const int n_tier = (int) (sizeof(ladder) / sizeof(ladder[0])); ++ int chosen = n_tier - 1; // floor default: even it may not fit → rolling-KV spills the rest ++ for (int i = 0; i < n_tier; ++i) { ++ if (per_cell_bytes(ladder[i].k, ladder[i].v) * (size_t) kv_size <= budget) { ++ chosen = i; ++ break; ++ } ++ } ++ if (chosen == 0) { ++ return; // f16 fits → no change, byte-identical boot ++ } ++ type_k = ladder[chosen].k; ++ type_v = ladder[chosen].v; ++ const size_t kv_bytes = per_cell_bytes(type_k, type_v) * (size_t) kv_size; ++ LLAMA_LOG_WARN("%s: auto KV tier = %s/%s (KV %.0f MiB %s budget %.0f MiB of %.0f MiB free); " ++ "set -ctk/-ctv to override\n", __func__, ++ ggml_type_name(type_k), ggml_type_name(type_v), ++ kv_bytes / 1048576.0, kv_bytes <= budget ? "fits" : "OVER (window spills)", ++ budget / 1048576.0, free_vram / 1048576.0); ++} ++ + llama_kv_cache::llama_kv_cache( + const llama_model & model, + ggml_type type_k, +@@ -442,6 +542,13 @@ llama_kv_cache::llama_kv_cache( + + const uint32_t n_layer_kv = hparams.n_layer_kv(); + ++ // opencoti HAL #582 P1 (#621) — auto KV-tier selection, run BEFORE any residency/allocation math ++ // consumes type_k/type_v (the auto-frac + window sizers below, and the per-layer tensor alloc). ++ // Opt-in (OPENCOTI_KV_AUTO_TIER=1) + user-default -ctk/-ctv + dense full-attention only; leaves ++ // type_k/type_v untouched (byte-identical) in every other case. See opencoti_auto_select_kv_tier. ++ opencoti_auto_select_kv_tier(model, hparams, type_k, type_v, v_trans, offload, ++ kv_size, n_stream, vram_target_mib, swa_type, filter); ++ + // opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md + // Resolve the effective HeadInfer fraction once for this cache. A negative + // incoming frac (OPENCOTI_HEADINFER_FRAC_AUTO) means "decide from free VRAM": diff --git a/patches/0117-rolling-kv-shift-rerope-639.patch b/patches/0117-rolling-kv-shift-rerope-639.patch new file mode 100644 index 0000000000000000000000000000000000000000..0bfbac3c15ce82f1b5ab54e6b6c0df669afca848 --- /dev/null +++ b/patches/0117-rolling-kv-shift-rerope-639.patch @@ -0,0 +1,165 @@ +opencoti patch 0117 — re-rope a spilled rolling-KV window on context-shift (#639, supersedes 0115 / bug-2148) + +0115 (rolling-kv-shift-guard) made llama_kv_cache::get_can_shift() REFUSE the shift whenever a +spilled position window was active: the k_shift graph viewed only k_per_stream (sized to the +window's wc cells, not get_size()), so a shift over-read the window tensor and asserted, and even +if the view fit the CPU tail would be left stale-roped. Refusing the shift cleanly disabled +ctx_shift AND KV prompt-cache-reuse (server-context.cpp:1093 reads get_can_shift at init) — which +silently broke context-shift, self-extend, and prompt-cache-reuse for any multi-session slot whose +KV spilled. That is exactly the serving-path capability this fork exists to keep working. + +0117 does the hard part instead of refusing it. build_graph_shift now re-ropes BOTH regions of a +spilled window with the SAME rope op (build_rope_shift): + + - WINDOW: GPU tensor k_per_stream[s], a view bounded to wc cells (not get_size()) so it never + over-reads. Its shift slice is k_shift_src[base + 0 .. base + wc). + - TAIL: host tensor k_cpu_per_stream[s], a view of the kv_size-wc tail cells. Its shift slice is + k_shift_src[base + wc .. base + kv_size). Because this tensor lives in host memory, ggml's + scheduler dispatches its rope op to the CPU backend automatically — the tail re-ropes with the + identical rope formula as the GPU window. Attention still runs entirely on GPU over the + already-roped window+tail, so this does NOT reintroduce the CPU-attention (CPU_FA_TAIL) + divergence class; the CPU only does a one-shot rope on shift events (never per decode token). + +get_can_shift() therefore returns true again (keeping the pre-existing STEP35 and n_pos_per_embd +guards); the 0115 refusal loop is removed. The per-cell shift deltas in k_shift_src cover every +logical cell regardless of physical placement, so each region views its contiguous slice. DCA +global layers keep shifting through the modular-delta channel (k_shift_dca); both regions use it. + +Fully-resident windows (wc == kv_size, no tail, window_cells sentinel 0) and every non-window +cache take the windowed==false path: the window view is sized get_size() and, for n_stream==1, +passes k_shift_src directly — BYTE-IDENTICAL to the pre-0117 op (verified). Host-only change; no +CUDA DSO rebuild (the tail rope is an existing ggml op dispatched to the CPU backend). + +Gate (bs2, Qwen2.5-14B-Instruct-1M-Q8_0, -c 12288, --context-shift --keep 256, GPU0, clean/no +contention). spilled (--vram-target 18000, 6912 resident / 5376 host tail = 1008 MiB): + - shift ENABLED (0 "shifting is not supported" logs), coherent through the shift: ZULU-9999 + present, longest monotonic run 1219, byte-identical output to the fully-resident control. + - decode-tps == pre-#639 baseline within noise (74.88->43.67 over 2823 tok vs baseline + 74.47->40.25): the re-rope adds no per-token cost (shift-event only). The earlier 26.9-tps + "regression" was a contended measurement (bug-2149), not this patch. + control (--vram-target 80000, fully resident): unchanged, shift fires normally, coherent. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2567,25 +2567,15 @@ + if (hparams.n_pos_per_embd() > 1) { + return false; + } +- // opencoti-hook: rolling-kv-shift-guard — see docs/features/rolling_kv_step_prefetch.md (#638, bug-2148) +- // A spilled position window (F5 M7 Stage 3c) stores the cache split across a GPU-resident +- // window [0,wc) and a CPU-resident tail [wc,kv_size). Context-shift (seq_add→pos_add) and +- // self-extend (seq_div→pos_div) re-rope the cache IN PLACE via the k_shift graph — but that +- // graph views only k_per_stream (sized wc, NOT get_size()) and has no path to re-rope the host +- // tail. Attempting a shift therefore (a) asserts in the k_shift view (ggml.c: data_size + +- // view_offs <= nbytes, over-reading the wc-cell window as get_size() cells) and (b) would leave +- // the tail stale-roped even if the view fit. Refuse the shift while a spilled window is active: +- // server-context.cpp:1093 reads this at init and cleanly disables ctx_shift / cache-reuse, so the +- // request is bounded at n_ctx instead of crashing. Fully-resident windows (wc==kv_size, no tail, +- // window_cells sentinel 0) and non-window caches are UNAFFECTED (loop finds no spilled layer). +- // The no-degradation alternative (re-rope the host tail on every shift) is deferred. +- for (const auto & layer : layers) { +- if (layer.window_cells > 0 +- && !layer.k_cpu_per_stream.empty() +- && layer.k_cpu_per_stream[0] != nullptr) { +- return false; +- } +- } ++ // opencoti-hook: rolling-kv-shift — see docs/features/rolling_kv_step_prefetch.md (#639, was #638/bug-2148) ++ // A spilled position window (F5 M7 Stage 3c) stores the cache split across a GPU-resident window ++ // [0,wc) and a CPU-resident tail [wc,kv_size). #639 makes build_graph_shift re-rope BOTH regions — ++ // the window on its GPU tensor (view bounded to wc, not get_size()) and the host tail on its CPU ++ // tensor. ggml dispatches each rope op to the backend the tensor lives on, so the tail re-ropes on ++ // the CPU backend with the SAME rope formula as the GPU window; attention still runs entirely on ++ // GPU over the already-roped window+tail (no CPU-attention divergence class). Context-shift ++ // (seq_add→pos_add), self-extend (seq_div→pos_div), and KV prompt-cache-reuse therefore all work on ++ // a spilled cache — a spilled window no longer refuses the shift. 0115 (the refusal) is superseded. + return true; + } + +@@ -4843,30 +4833,71 @@ + + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + ++ // opencoti-hook: rolling-kv-shift — see docs/features/rolling_kv_step_prefetch.md (#639). ++ // A spilled position window (F5 M7 Stage 3c) stores logical cells [0,wc) in the GPU tensor ++ // k_per_stream (sized wc) and cells [wc,kv_size) in the host tensor k_cpu_per_stream (sized ++ // kv_size-wc). Physical window cell i == logical cell i; physical tail cell j == logical cell ++ // wc+j (append-only window class, swa_type==NONE — the only class the sizer windows). To ++ // re-rope a shifted cache we re-rope BOTH regions with the SAME rope op (build_rope_shift): ++ // the window on its GPU tensor, the tail on its CPU tensor. ggml's scheduler dispatches each ++ // rope op to the backend the tensor lives on, so the tail re-ropes on the CPU backend with the ++ // identical rope formula; attention still runs entirely on GPU over the already-roped ++ // window+tail (no CPU-attention divergence class, cf. CPU_FA_TAIL). The per-cell shift deltas ++ // are in k_shift_src[cell] for ALL logical cells regardless of physical placement, so each ++ // region views its contiguous slice: window [base+0, base+wc), tail [base+wc, base+kv_size). + for (uint32_t s = 0; s < n_stream; ++s) { +- ggml_tensor * layer_k = layer.k_per_stream[s]; +- ggml_tensor * k = +- ggml_view_3d(ctx, layer_k, +- n_rot, n_head_kv, get_size(), +- ggml_row_size(layer_k->type, n_embd_head_k), +- ggml_row_size(layer_k->type, n_embd_k_gqa), +- ggml_row_size(layer_k->type, n_embd_nope)); ++ const uint32_t wc = layer.window_cells; ++ const bool windowed = wc > 0 ++ && !layer.k_cpu_per_stream.empty() ++ && layer.k_cpu_per_stream[s] != nullptr; ++ const uint32_t n_win = windowed ? wc : (uint32_t) get_size(); ++ const uint32_t n_tail = windowed && get_size() > wc ? (uint32_t) get_size() - wc : 0; + + // opencoti F5 dca (#593, T87.pD-gen #617): DCA global layers store PRE-roped K, so they + // shift via the modular-delta channel (k_shift_dca); SWA / non-DCA layers stay on the + // native absolute k_shift. The DCA channel is only allocated when this cache owns a DCA +- // layer — exactly when the predicate below can be true. ++ // layer — exactly when the predicate below can be true. Both regions use the same channel. + ggml_tensor * k_shift_src = (cparams.dca_enabled && !hparams.is_swa(il)) + ? inp->k_shift_dca + : inp->k_shift; +- ggml_tensor * k_shift_s = (n_stream == 1) +- ? k_shift_src +- : ggml_view_1d(ctx, k_shift_src, get_size(), +- (size_t) s * get_size() * k_shift_src->nb[0]); ++ const size_t base = (size_t) s * get_size(); // element offset of this stream's shift slice ++ ++ // WINDOW region — GPU tensor k_per_stream[s], logical cells [0,n_win). ++ { ++ ggml_tensor * layer_k = layer.k_per_stream[s]; ++ ggml_tensor * k = ++ ggml_view_3d(ctx, layer_k, ++ n_rot, n_head_kv, n_win, ++ ggml_row_size(layer_k->type, n_embd_head_k), ++ ggml_row_size(layer_k->type, n_embd_k_gqa), ++ ggml_row_size(layer_k->type, n_embd_nope)); ++ ++ // Preserve the exact pre-#639 op for the non-windowed unified path (byte-identical): ++ // pass k_shift_src directly when n_stream==1 AND not windowed. Otherwise a slice view. ++ ggml_tensor * k_shift_s = (n_stream == 1 && !windowed) ++ ? k_shift_src ++ : ggml_view_1d(ctx, k_shift_src, n_win, base * k_shift_src->nb[0]); ++ ++ ggml_tensor * cur = build_rope_shift(cparams, ctx, k, k_shift_s, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); ++ ggml_build_forward_expand(gf, cur); ++ } ++ ++ // TAIL region — host tensor k_cpu_per_stream[s], logical cells [wc, kv_size). Runs on the ++ // CPU backend automatically (the tensor lives there). Only present on a spilled window. ++ if (windowed && n_tail > 0) { ++ ggml_tensor * layer_kc = layer.k_cpu_per_stream[s]; ++ ggml_tensor * kt = ++ ggml_view_3d(ctx, layer_kc, ++ n_rot, n_head_kv, n_tail, ++ ggml_row_size(layer_kc->type, n_embd_head_k), ++ ggml_row_size(layer_kc->type, n_embd_k_gqa), ++ ggml_row_size(layer_kc->type, n_embd_nope)); + +- ggml_tensor * cur = build_rope_shift(cparams, ctx, k, k_shift_s, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); ++ ggml_tensor * k_shift_t = ggml_view_1d(ctx, k_shift_src, n_tail, (base + wc) * k_shift_src->nb[0]); + +- ggml_build_forward_expand(gf, cur); ++ ggml_tensor * cur = build_rope_shift(cparams, ctx, kt, k_shift_t, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il); ++ ggml_build_forward_expand(gf, cur); ++ } + } + } + diff --git a/patches/0118-kv-auto-tier-tstar-spill.patch b/patches/0118-kv-auto-tier-tstar-spill.patch new file mode 100644 index 0000000000000000000000000000000000000000..59801dc70c72197396363ef2c23d9e4d18c8b8e5 --- /dev/null +++ b/patches/0118-kv-auto-tier-tstar-spill.patch @@ -0,0 +1,291 @@ +opencoti patch 0118 — T*-aware auto rolling-KV spill (HAL #582 P1+ / #621 follow-on) + +Extends the 0116 auto KV-tier helper (`opencoti_auto_select_kv_tier`) so it can +KEEP a higher-quality tier (up to f16) with a SMALL host-tail spill when that +spill is provably cheap on this host + model + context — instead of always +quantizing to the first fully-resident rung. Opt-in under +`OPENCOTI_KV_AUTO_TIER=1`; byte-identical when the env is unset (the gate +returns before any measurement). HOST-ONLY: the FA compute microbench dispatches +to the existing CUDA DSO, so no ggml-cuda.so rebuild. + +MECHANISM (all additive, in src/llama-kv-cache.cpp; opencoti-hook: kv-auto-tier). + + T1 — explicit-axis handling (req #4). The incoming type_k/type_v already encode + operator intent (auto runs BEFORE anything else mutates them): an explicit + `-ctv` disables auto-tier entirely; an explicit `-ctk` holds K and walks only + the V axis; both-f16 gets the full ladder. iSWA/SWA caches are left alone. + + T2 — bandwidth probes (req #3: RAM->GPU bandwidth, not just PCIe). A host-side + one-thread `memcpy` bench measures host DRAM read GB/s (`host_ram_bw_gbps`, + static-cached, env `OPENCOTI_KV_HOST_RAM_GBPS`); combined with the existing + ctor `pcie_bw_gbps` as `bw_eff = min(pcie, host_ram)`. On a fast-PCIe host + (bs2 PCIe5 + DDR5) the DDR read binds; without this probe bw_eff is ~5x too + optimistic there. VRAM bandwidth is subsumed by the FA microbench (which reads + KV from VRAM), so no separate device-memcpy probe is needed. + + T3 — c_engage boot micro-measure (req B). Reuses the FA compute microbench + (opencoti_fa_compute_probe_ms, #351) at two cell counts (8192 window vs 256 + floor) to derive `fa_ms_cell` and the fixed streaming-engagement cost + `c_engage` (env `OPENCOTI_KV_CENGAGE_MS`), captured in the live DCA state. + + T4 — T* drop-model fit-condition. The 0116 ladder walk changes from "first + tier fully resident" to "first tier whose SPILL <= T*_bytes", where + T*_bytes = (target_drop * t_resident - c_engage) * bw_eff, target_drop default + 0.20 (env `OPENCOTI_KV_TSTAR_DROP`), clamped >=0 and <= a cliff (env + `OPENCOTI_KV_TSTAR_MAX_SPILL_MIB` default 800 MiB, the conservative DCA-on + pre-cliff band). A fast-path guard keeps the common "f16 fits" boot + byte-identical (skips the probe). When the probe is unavailable or T*<=0 the + gate degrades exactly to the 0116 "fully resident" behavior. Short ctx => + tiny t_resident => c_engage dominates => T*<=0 => quantize (the ctx-value gate, + req #2, falls out of the formula). All diagnostics are WARN-level, mirroring + the existing `auto KV tier` WARN, for operator auditability. + +VALIDATED (docs/features/tstar_auto_spill.md). T2-T4 terms confirmed firing on +both hosts (solidPC PCIe-bound bw_eff 6.7; bs2 host-RAM-bound 9.3-10.3), drop +model monotone in target_drop. T5 RULER-niah quality/tps gate PASS (bs2 GPU0, +Qwen2.5-14B-1M Q8_0, ctx 65536): at a ~2 GB spill, auto correctly selects +q8_0/q8_0 (T* only ~14 MiB), f16-window-spill decode-tps drops 78.5% (>> 20% +target), q8 niah == f16 niah == 100 (lossless), and q8 tps (35.3) 3.3x the +f16-spill (10.8) -- the quantize verdict is correct and the fallback strictly +dominates. Correctness via RULER niah, never greedy needle. + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index e1f723f..ffdac1d 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -355,6 +355,58 @@ static uint32_t opencoti_compute_resident_window_cells( + return cells; + } + ++// opencoti #582 T3 — forward decl of the streaming-FA compute microbench (defined below near ++// build_rolling_kv_plan). The auto-tier T* fit-condition calls it BEFORE allocation to MEASURE the ++// resident per-cell FA cost (t_resident) and the fixed launch floor (c_engage proxy). It builds a ++// transient backend + synthetic decode-shape FA over `bench_cells` keys and never aborts boot ++// (returns 0.0f on any shortfall). See docs/features/tstar_auto_spill.md. ++static float opencoti_fa_compute_probe_ms( ++ const llama_model & model, const llama_hparams & hparams, ++ int32_t il, ggml_type type_k, ggml_type type_v, int64_t bench_cells, bool on_cpu); ++ ++// opencoti #582 T2 — host DRAM streaming bandwidth (GB/s), measured once per process with a plain ++// memcpy bench. Combined with the PCIe bandwidth as bw_eff = min(pcie, host_ram) so the T* spill ++// model reflects RAM->GPU cost, not just the PCIe link (req #3: DDR4 vs DDR5 matters). One-way copy ++// GB/s is the effective host-stream figure. Overridable via OPENCOTI_KV_HOST_RAM_GBPS. ++static float opencoti_host_ram_bw_gbps() { ++ static float cached = -1.0f; ++ if (cached >= 0.0f) { ++ return cached; ++ } ++ const char * env = getenv("OPENCOTI_KV_HOST_RAM_GBPS"); ++ if (env != nullptr) { ++ cached = (float) atof(env); ++ return cached; ++ } ++ const size_t N = (size_t) 64 * 1024 * 1024; ++ char * a = (char *) malloc(N); ++ char * b = (char *) malloc(N); ++ if (a == nullptr || b == nullptr) { ++ free(a); free(b); ++ cached = 0.0f; ++ return cached; ++ } ++ memset(a, 1, N); ++ memcpy(b, a, N); // warm the pages ++ const int reps = 6; ++ volatile uint64_t sink = 0; // defeat dead-store elimination (else the whole loop is elided) ++ const auto t0 = std::chrono::high_resolution_clock::now(); ++ for (int i = 0; i < reps; ++i) { ++ // Mutate the source each rep so identical copies can't be collapsed, and read the ++ // destination into a volatile sink so the memcpy is provably live (not a dead store). ++ a[((size_t) i * 4096) % N] = (char) (i + 1); ++ memcpy(b, a, N); ++ sink += (uint64_t) (unsigned char) b[((size_t) i * 4096) % N]; ++ } ++ const auto t1 = std::chrono::high_resolution_clock::now(); ++ (void) sink; ++ const double secs = std::chrono::duration(t1 - t0).count(); ++ const double gb = (double) N * reps / 1e9; ++ cached = secs > 0.0 ? (float) (gb / secs) : 0.0f; ++ free(a); free(b); ++ return cached; ++} ++ + // opencoti-hook: kv-auto-tier — see docs/evaluations/hal_kv_compression.md §5 P1 (#621, HAL #582) + // Auto KV-tier selection. When opted in (env OPENCOTI_KV_AUTO_TIER=1) and the user LEFT -ctk/-ctv + // at the f16 default, pick the LEAST-compressing scalar KV type pair that keeps the WHOLE cache +@@ -370,7 +422,7 @@ static void opencoti_auto_select_kv_tier( + const llama_model & model, const llama_hparams & hparams, + ggml_type & type_k, ggml_type & type_v, + bool v_trans, bool offload, uint32_t kv_size, uint32_t n_stream, +- uint32_t vram_target_mib, llama_swa_type swa_type, ++ uint32_t vram_target_mib, llama_swa_type swa_type, float pcie_bw_gbps, + const std::function & filter) { + if (getenv("OPENCOTI_KV_AUTO_TIER") == nullptr) { + return; // opt-in only — silent + byte-identical when unset +@@ -378,8 +430,15 @@ static void opencoti_auto_select_kv_tier( + if (!offload || kv_size == 0) { + return; // no GPU offload → no residency pressure to relieve + } +- if (type_k != GGML_TYPE_F16 || type_v != GGML_TYPE_F16) { +- return; // user set -ctk/-ctv explicitly → respect the override ++ // opencoti #582 T1 (req #4): honor an explicit -ctk/-ctv as a block. Auto-tier runs BEFORE it ++ // mutates type_k/type_v, and nothing else makes them non-f16 at this point, so a non-f16 INCOMING ++ // type == the user passed that flag. -ctv explicit → auto disabled (user owns the V axis / the ++ // whole pair; only the rolling-KV window stays auto). -ctk explicit (V still f16) → HOLD K and ++ // walk only the V axis. See docs/features/tstar_auto_spill.md. ++ const bool k_explicit = (type_k != GGML_TYPE_F16); ++ const bool v_explicit = (type_v != GGML_TYPE_F16); ++ if (v_explicit) { ++ return; // -ctv set → auto disabled + } + if (swa_type != LLAMA_SWA_TYPE_NONE) { + return; // dense full-attention is the validated class; iSWA keeps f16 +@@ -387,11 +446,13 @@ static void opencoti_auto_select_kv_tier( + + // Budget — identical math to opencoti_compute_resident_window_cells (type-independent). + ggml_backend_dev_t dev = nullptr; ++ int32_t il0 = -1; // opencoti #582 T4 — first KV layer, for the FA compute microbench + for (uint32_t il = 0; il < hparams.n_layer; ++il) { + if (!hparams.has_kv(il) || (filter && !filter(il))) { + continue; + } + dev = model.dev_layer(il); ++ il0 = (int32_t) il; + break; + } + if (!dev) { +@@ -428,22 +489,121 @@ static void opencoti_auto_select_kv_tier( + // Ladder: least → most compression (HAL §5 P1 / §2 quality bar). q8_0/q4_0 is the + // recommended hi-K/cheap-V recipe (§2); q5_1/q4_0 is the last-resort floor. + struct kv_tier { ggml_type k, v; }; +- static const kv_tier ladder[] = { ++ // Full ladder (least→most compression). When -ctk pinned K (k_explicit), HOLD K and walk only ++ // the V axis (V-ladder f16 → q8_0 → q4_0) so the explicit K is honored as a block (req #4). ++ // ladder_heldk is runtime-initialized (uses type_k) → not static. ++ static const kv_tier ladder_full[] = { + { GGML_TYPE_F16, GGML_TYPE_F16 }, + { GGML_TYPE_Q8_0, GGML_TYPE_Q8_0 }, + { GGML_TYPE_Q8_0, GGML_TYPE_Q4_0 }, + { GGML_TYPE_Q5_1, GGML_TYPE_Q4_0 }, + }; +- const int n_tier = (int) (sizeof(ladder) / sizeof(ladder[0])); +- int chosen = n_tier - 1; // floor default: even it may not fit → rolling-KV spills the rest ++ const kv_tier ladder_heldk[] = { ++ { type_k, GGML_TYPE_F16 }, ++ { type_k, GGML_TYPE_Q8_0 }, ++ { type_k, GGML_TYPE_Q4_0 }, ++ }; ++ const kv_tier * ladder = k_explicit ? ladder_heldk : ladder_full; ++ const int n_tier = k_explicit ++ ? (int) (sizeof(ladder_heldk) / sizeof(ladder_heldk[0])) ++ : (int) (sizeof(ladder_full) / sizeof(ladder_full[0])); ++ // opencoti #582 T4 fast path — if the top (least-compressing) tier already FULLY fits, there is ++ // no spill to weigh: keep the user's types (byte-identical) and skip the boot FA microbench. The ++ // T* drop-model below only runs when the top tier spills (the case where f16-window vs quantize ++ // is a real decision), so short-ctx / roomy-VRAM boots pay zero extra probe cost. ++ const size_t top_kv_bytes = per_cell_bytes(ladder[0].k, ladder[0].v) * (size_t) kv_size; ++ LLAMA_LOG_WARN("%s: budget %.0f MiB (of %.0f MiB free) — top tier %s/%s KV %.0f MiB (%s by %.0f MiB)\n", ++ __func__, budget / 1048576.0, free_vram / 1048576.0, ++ ggml_type_name(ladder[0].k), ggml_type_name(ladder[0].v), top_kv_bytes / 1048576.0, ++ top_kv_bytes <= budget ? "fits, margin" : "SPILLS", ++ (top_kv_bytes <= budget ? (budget - top_kv_bytes) : (top_kv_bytes - budget)) / 1048576.0); ++ if (top_kv_bytes <= budget) { ++ return; ++ } ++ // opencoti #582 T4 — T* drop-model fit condition. Rather than "first tier that FULLY fits", ++ // accept the first (least-compressing) tier whose SPILL past the resident budget is within ++ // T*_bytes: the tail whose streaming cost keeps the per-token decode drop <= target (default 20%, ++ // user 2026-07-11). This lets f16-window+spill beat quantizing when the spill is provably cheap on ++ // THIS host+model+ctx (Policy C). Terms are MEASURED at boot (req B): t_resident from the FA ++ // compute microbench (#351), bw_eff = min(pcie, host_ram) (req #3), and a per-tier/ctx t_resident ++ // (req #2 ctx-gate falls out — bigger ctx ⇒ bigger spill ⇒ exceeds T*). When the probe is ++ // unavailable or T*<=0 (short-ctx / slow link), T*_bytes collapses to 0 ⇒ identical to the ++ // "fully resident" gate ⇒ safe by default. See docs/features/tstar_auto_spill.md. ++ const double target_drop = [] { ++ const char * e = getenv("OPENCOTI_KV_TSTAR_DROP"); ++ const double d = e ? atof(e) : 0.20; ++ return (d > 0.0 && d < 1.0) ? d : 0.20; ++ }(); ++ const int64_t n_layer_kv = (int64_t) std::max(1u, hparams.n_layer_kv()); ++ // Measured GPU FA ms per resident cell (whole model) + fixed launch floor (c_engage proxy). ++ double fa_ms_cell = 0.0, c_engage_ms = 0.0; ++ { ++ const float ms_win = opencoti_fa_compute_probe_ms(model, hparams, il0, GGML_TYPE_F16, GGML_TYPE_F16, 8192, false); ++ const float ms_floor = opencoti_fa_compute_probe_ms(model, hparams, il0, GGML_TYPE_F16, GGML_TYPE_F16, 256, false); ++ if (ms_win > 0.0f) { ++ fa_ms_cell = (double) ms_win / 8192.0 * (double) n_layer_kv; ++ const double floor_layer = std::max(0.0, (double) ms_floor - (double) ms_win / 8192.0 * 256.0); ++ c_engage_ms = floor_layer * (double) n_layer_kv; ++ } ++ const char * ce = getenv("OPENCOTI_KV_CENGAGE_MS"); ++ if (ce) c_engage_ms = atof(ce); ++ } ++ const float host_ram_gbps = opencoti_host_ram_bw_gbps(); ++ double bw_eff_gbps = host_ram_gbps > 0.0f ? (double) host_ram_gbps : 0.0; ++ if (pcie_bw_gbps > 0.0f) { ++ bw_eff_gbps = bw_eff_gbps > 0.0 ? std::min(bw_eff_gbps, (double) pcie_bw_gbps) : (double) pcie_bw_gbps; ++ } ++ const double bw_eff_bytes_per_ms = bw_eff_gbps * 1e6; // GB/s -> bytes/ms ++ // Per-tier tolerable spill. t_resident scales with the tier's resident-cell count (cheaper tier ++ // fits more cells ⇒ larger t_resident ⇒ larger T*). Clamped to the empirical rolling-KV spill ++ // cliff (req #5: DCA-on knees earlier ~800 MiB on bs2; DCA-off plateau ~1024 MiB) so T* never ++ // recommends a spill past the point rolling-KV stays effective. ++ auto tstar_bytes = [&](size_t pcb) -> double { ++ if (fa_ms_cell <= 0.0 || bw_eff_bytes_per_ms <= 0.0 || pcb == 0 || budget == 0) { ++ return 0.0; ++ } ++ const double resident_cells = (double) budget / (double) pcb; ++ const double t_resident_ms = fa_ms_cell * resident_cells; ++ const double tail_ms = target_drop * t_resident_ms - c_engage_ms; ++ if (tail_ms <= 0.0) { ++ return 0.0; ++ } ++ const double tb = tail_ms * bw_eff_bytes_per_ms; ++ // req #5 — default to the conservative DCA-on pre-cliff band (~800 MiB observed on bs2); it is ++ // safe for DCA-off too (whose plateau extends further, ~1024 MiB). Relax via env for DCA-off. ++ const char * e = getenv("OPENCOTI_KV_TSTAR_MAX_SPILL_MIB"); ++ const double cliff_mib = e ? atof(e) : 800.0; ++ return std::min(tb, cliff_mib * 1048576.0); ++ }; ++ // Operator visibility (req B): the MEASURED terms behind the T* spill budget, plus the resulting ++ // tolerable spill for the TOP tier vs its actual spill (the flip test). fa_ms_cell==0 ⇒ the FA ++ // microbench fell short (transient backend / synthetic-graph shortfall) ⇒ T*_bytes collapses to 0 ++ // ⇒ this decision degrades to the T1 "fully resident" gate. See docs/features/tstar_auto_spill.md. ++ LLAMA_LOG_WARN("%s: T* drop-model — fa_ms/cell=%.4g c_engage=%.2fms bw_eff=%.1f GB/s " ++ "(host_ram=%.1f pcie=%.1f) target_drop=%.2f → top tier tolerable spill %.0f MiB " ++ "(actual spill %.0f MiB)\n", __func__, fa_ms_cell, c_engage_ms, bw_eff_gbps, ++ (double) host_ram_gbps, (double) pcie_bw_gbps, target_drop, ++ tstar_bytes(per_cell_bytes(ladder[0].k, ladder[0].v)) / 1048576.0, ++ (double) (top_kv_bytes - budget) / 1048576.0); ++ int chosen = n_tier - 1; // floor default: even it may not fit within T* → rolling-KV spills the rest + for (int i = 0; i < n_tier; ++i) { +- if (per_cell_bytes(ladder[i].k, ladder[i].v) * (size_t) kv_size <= budget) { ++ const size_t pcb = per_cell_bytes(ladder[i].k, ladder[i].v); ++ if ((double) (pcb * (size_t) kv_size) <= (double) budget + tstar_bytes(pcb)) { + chosen = i; + break; + } + } + if (chosen == 0) { +- return; // f16 fits → no change, byte-identical boot ++ // tier 0 (f16, or held-K + f16-V) accepted → type unchanged → byte-identical. If it was ++ // accepted via T* (spills but within tolerance) announce it so Policy C is visible. ++ const size_t kvb0 = per_cell_bytes(ladder[0].k, ladder[0].v) * (size_t) kv_size; ++ if (kvb0 > budget) { ++ LLAMA_LOG_WARN("%s: auto KV tier = %s/%s KEPT (T* spill %.0f MiB within tol; budget %.0f MiB " ++ "of %.0f MiB free) — f16-window preferred over quantizing; set -ctk/-ctv to override\n", ++ __func__, ggml_type_name(ladder[0].k), ggml_type_name(ladder[0].v), ++ (kvb0 - budget) / 1048576.0, budget / 1048576.0, free_vram / 1048576.0); ++ } ++ return; + } + type_k = ladder[chosen].k; + type_v = ladder[chosen].v; +@@ -547,7 +707,7 @@ llama_kv_cache::llama_kv_cache( + // Opt-in (OPENCOTI_KV_AUTO_TIER=1) + user-default -ctk/-ctv + dense full-attention only; leaves + // type_k/type_v untouched (byte-identical) in every other case. See opencoti_auto_select_kv_tier. + opencoti_auto_select_kv_tier(model, hparams, type_k, type_v, v_trans, offload, +- kv_size, n_stream, vram_target_mib, swa_type, filter); ++ kv_size, n_stream, vram_target_mib, swa_type, pcie_bw_gbps, filter); + + // opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md + // Resolve the effective HeadInfer fraction once for this cache. A negative diff --git a/patches/0119-p0-dcaon-scalar-inregister.patch b/patches/0119-p0-dcaon-scalar-inregister.patch new file mode 100644 index 0000000000000000000000000000000000000000..de20323671a09ba3e2441b020bb22e997d10733c --- /dev/null +++ b/patches/0119-p0-dcaon-scalar-inregister.patch @@ -0,0 +1,1220 @@ +opencoti patch 0119 — P0 DCA-on scalar in-register KV (#582-P0 / #643 beachhead + #644 full matrix) + +Kills the whole-cache to_fp16_nc dequant-lift tax that dominated quant-KV decode +under DCA-on (HAL §3b profile: ~29.5 s/decode of dequantize_block; the FA kernel +itself is already f16-identical to the f16-KV cell). Scalar q4_0/q4_1/q5_0/q5_1/ +q6_0/q8_0 K/V are now read IN-REGISTER inside the multichunk `dca_fused` MMA +kernel, reusing the SAME generic load_tile_dequant / get_dequantize_V path the +turbo tiers already use (WS2 #629) — no new kernel math. This is the scalar +generalization of #629's turbo-V in-register read. + +RESULTS (bs2, Qwen2.5-14B-1M): + #643 beachhead q8_0-K/q4_0-V @D128 ctx128k : real_frac 0.0 (bit-identical to the + shipped materialize path) + 2.13x decode (14.72 -> 31.33 tps). + #644 full matrix @D128, 32k multichunk DCA-on: all 10 pairs real_frac 0.0 — + symmetric diagonal {q4_0,q4_1,q5_0,q5_1,q6_0,q8_0} K==V + hi-K/cheap-V + {q8K/{q4,q5,q6}V, q6K/q4V}. + +MECHANISM (all additive; opencoti-hook: P0 DCA-on scalar in-register): + - fattn-common.cuh: the q4_1/q5_0/q5_1/q6_0 V-readers gain an ne==8 branch (two + proven ne==4 sub-groups; an 8-aligned chunk never crosses a 32-elem block, and + the 5-bit qh is read <4,...> = full 4-byte mask, no overread) so the MMA loader + (fixed at ne_per_chunk=8, shared with the turbo path — NOT changed) can serve + them. q4_0 already had ne==8. This is the only correctness surface (bug-2152). + - fattn-mma-f16.cuh: the dca_fused kernel skips the in-launch K materialize when + instantiated for a scalar type_K (reads raw quant K in-register), mirroring the + existing type_V==F16 guard; + externs for the D128 scalar instances (10 pairs + x 16 ncols configs). + - fattn.cu: decode-gated dispatch (Q->ne[1] <= WS2_NQ_MAX, default 16; env + WS2_DCA_SCALAR_KV, default on). A matched (decode, scalar) pair dispatches its + typed instance; any other combo falls through to the shipped F16-materialize + default (large-n_q prefill keeps the fast in-launch lift, bug-720). No behavior + change when WS2_DCA_SCALAR_KV=0. + - 40 new template-instance TUs (fattn-mma-f16-dca-fused-scalar--d128-{0..3}.cu; + 4 ncols2 shards x 10 pairs). Named fattn-mma-*.cu so collect_gpu_sources globs + them (build-functions.sh). D256/D512 scalar deferred (Gemma KV is turbo, already + in-register). DSO grows 2.5->3.83GB, absorbed by the fatbin-last linker fragment + (bug-2140); -mcmodel=large stays OFF. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +index e2cde24..8020375 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +@@ -445,6 +445,14 @@ static __device__ __forceinline__ void dequantize_V_bf16(const void * __restrict + } + } + ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — the ne<=4 body is the shipped VEC-path ++// reader (byte-identical). ne==8 is NEW: the dca_fused MMA loader (flash_attn_ext_f16_load_tile_dequant) ++// reads V in 8-element chunks (ne_per_chunk = 2*h2_per_chunk = 8), which the original q4_0 reader could not ++// serve (it packs `ne` bytes into one 4-byte int → caps at ne=4). On THIS path i0 is always a multiple of 8 ++// and QK4_0/2 == 16 is a multiple of 8, so `shift` (low- vs high-nibble half) is CONSTANT across the whole ++// 8-chunk (0..7, 8..15 → shift 0; 16..23, 24..31 → shift 1; never straddles the block half-boundary). So an ++// ne==8 read is just two independent 4-byte q4_0 groups with the same shift — no new dequant math, only a ++// wider fetch. iqs+sub+3 <= 8+4+3 = 15 < 16 ⇒ always in-block. + template + static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q4_0 * x = (const block_q4_0 *) vx; +@@ -453,34 +461,68 @@ static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict + const int iqs = i0 % (QK4_0/2); + const int shift = (i0 % QK4_0) / (QK4_0/2); + +- int q; +- static_assert(ne == 2 || ne == 4, "bad ne"); +- ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +- q >>= 4*shift; +- q &= 0x0F0F0F0F; +- q = __vsubss4(q, 0x08080808); ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ if constexpr (ne <= 4) { ++ int q; ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; ++ q = __vsubss4(q, 0x08080808); + +- const int8_t * q8 = (const int8_t *) &q; ++ const int8_t * q8 = (const int8_t *) &q; + + #ifdef FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const half2 d = __half2half2(x[ib].d); ++ if constexpr (std::is_same_v) { ++ const half2 d = __half2half2(x[ib].d); + + #pragma unroll +- for (int l0 = 0; l0 < ne; l0 += 2) { +- ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ for (int l0 = 0; l0 < ne; l0 += 2) { ++ ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ } ++ } else ++#endif // FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ const float d = x[ib].d; ++ ++#pragma unroll ++ for (int l = 0; l < ne; ++l) { ++ ((float *) dst)[l] = d * q8[l]; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); + } +- } else ++ } else { // ne == 8: two 4-byte q4_0 groups, shift constant across the chunk (see header note) ++#pragma unroll ++ for (int sub = 0; sub < ne; sub += 4) { ++ int q; ++ ggml_cuda_memcpy_1<4, 2>(&q, x[ib].qs + iqs + sub); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; ++ q = __vsubss4(q, 0x08080808); ++ ++ const int8_t * q8 = (const int8_t *) &q; ++ ++#ifdef FP16_AVAILABLE ++ if constexpr (std::is_same_v) { ++ const half2 d = __half2half2(x[ib].d); ++ ++#pragma unroll ++ for (int l0 = 0; l0 < 4; l0 += 2) { ++ ((half2 *) dst)[(sub + l0)/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ } ++ } else + #endif // FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const float d = x[ib].d; ++ if constexpr (std::is_same_v) { ++ const float d = x[ib].d; + + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- ((float *) dst)[l] = d * q8[l]; ++ for (int l = 0; l < 4; ++l) { ++ ((float *) dst)[sub + l] = d * q8[l]; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); ++ } + } +- } else { +- static_assert(std::is_same_v, "bad type"); + } + } + +@@ -488,39 +530,47 @@ template + static __device__ __forceinline__ void dequantize_V_q4_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q4_1 * x = (const block_q4_1 *) vx; + +- const int64_t ib = i0 / QK4_1; +- const int iqs = i0 % (QK4_1/2); +- const int shift = (i0 % QK4_1) / (QK4_1/2); ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ // opencoti-hook: P0 DCA-on scalar in-register (#644) — ne==8 = two proven ne=4 groups (sub∈{0,4}) at a ++ // 4-aligned sub-index (an ne=8 chunk is 8-aligned within a 32-block so never crosses a block). Each group ++ // is byte-identical to the ne<=4 body below at i0+sub; keeps the shared MMA loader (ne_per_chunk=8) intact. ++#pragma unroll ++ for (int sub = 0; sub < ne; sub += (ne <= 4 ? ne : 4)) { ++ const int64_t is = i0 + sub; ++ const int64_t ib = is / QK4_1; ++ const int iqs = is % (QK4_1/2); ++ const int shift = (is % QK4_1) / (QK4_1/2); ++ constexpr int nl = ne <= 4 ? ne : 4; + +- int q; +- static_assert(ne == 2 || ne == 4, "bad ne"); +- ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +- q >>= 4*shift; +- q &= 0x0F0F0F0F; ++ int q; ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; + +- const int8_t * q8 = (const int8_t *) &q; ++ const int8_t * q8 = (const int8_t *) &q; + + #ifdef FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const half2 dm = x[ib].dm; +- const half2 d = __half2half2( __low2half(dm)); +- const half2 m = __half2half2(__high2half(dm)); ++ if constexpr (std::is_same_v) { ++ const half2 dm = x[ib].dm; ++ const half2 d = __half2half2( __low2half(dm)); ++ const half2 m = __half2half2(__high2half(dm)); + + #pragma unroll +- for (int l0 = 0; l0 < ne; l0 += 2) { +- ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; +- } +- } else ++ for (int l0 = 0; l0 < nl; l0 += 2) { ++ ((half2 *) dst)[(sub + l0)/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; ++ } ++ } else + #endif // FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const float2 dm = __half22float2(x[ib].dm); ++ if constexpr (std::is_same_v) { ++ const float2 dm = __half22float2(x[ib].dm); + + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- ((float *) dst)[l] = dm.x * q8[l] + dm.y; ++ for (int l = 0; l < nl; ++l) { ++ ((float *) dst)[sub + l] = dm.x * q8[l] + dm.y; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); + } +- } else { +- static_assert(std::is_same_v, "bad type"); + } + } + +@@ -528,49 +578,57 @@ template + static __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q5_0 * x = (const block_q5_0 *) vx; + +- const int64_t ib = i0 / QK5_0; +- const int idq = i0 % QK5_0; +- const int iqs = i0 % (QK5_0/2); +- const int shift = (i0 % QK5_0) / (QK5_0/2); +- +- int q; +- static_assert(ne == 2 || ne == 4, "bad ne"); +- ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +- q >>= 4*shift; +- q &= 0x0F0F0F0F; ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ // opencoti-hook: P0 DCA-on scalar in-register (#644) — ne==8 = two proven ne=4 groups (sub∈{0,4}). Each ++ // group re-reads the FULL 4-byte qh via ggml_cuda_memcpy_1 (nl=4 at ne==8), so no qh overread; idq+l ++ // stays ≤31 (is is 4-aligned). Byte-identical to the ne<=4 body per sub. Shared MMA loader untouched. ++#pragma unroll ++ for (int sub = 0; sub < ne; sub += (ne <= 4 ? ne : 4)) { ++ const int64_t is = i0 + sub; ++ const int64_t ib = is / QK5_0; ++ const int idq = is % QK5_0; ++ const int iqs = is % (QK5_0/2); ++ const int shift = (is % QK5_0) / (QK5_0/2); ++ constexpr int nl = ne <= 4 ? ne : 4; ++ ++ int q; ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; + +- { +- int qh; +- ggml_cuda_memcpy_1(&qh, x[ib].qh); ++ { ++ int qh; ++ ggml_cuda_memcpy_1(&qh, x[ib].qh); + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4); ++ for (int l = 0; l < nl; ++l) { ++ q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4); ++ } + } +- } + +- q = __vsubss4(q, 0x10101010); ++ q = __vsubss4(q, 0x10101010); + +- const int8_t * q8 = (const int8_t *) &q; ++ const int8_t * q8 = (const int8_t *) &q; + + #ifdef FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const half2 d = __half2half2(x[ib].d); ++ if constexpr (std::is_same_v) { ++ const half2 d = __half2half2(x[ib].d); + + #pragma unroll +- for (int l0 = 0; l0 < ne; l0 += 2) { +- ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); +- } +- } else ++ for (int l0 = 0; l0 < nl; l0 += 2) { ++ ((half2 *) dst)[(sub + l0)/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ } ++ } else + #endif // FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const float d = x[ib].d; ++ if constexpr (std::is_same_v) { ++ const float d = x[ib].d; + + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- ((float *) dst)[l] = d * q8[l]; ++ for (int l = 0; l < nl; ++l) { ++ ((float *) dst)[sub + l] = d * q8[l]; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); + } +- } else { +- static_assert(std::is_same_v, "bad type"); + } + } + +@@ -583,54 +641,62 @@ template + static __device__ __forceinline__ void dequantize_V_q6_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q6_0 * x = (const block_q6_0 *) vx; + +- const int64_t ib = i0 / QK6_0; +- const int idq = i0 % QK6_0; +- const int iqs = i0 % (QK6_0/2); +- const int shift = (i0 % QK6_0) / (QK6_0/2); +- +- int q; +- static_assert(ne == 2 || ne == 4, "bad ne"); +- ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +- q >>= 4*shift; +- q &= 0x0F0F0F0F; ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ // opencoti-hook: P0 DCA-on scalar in-register (#644) — ne==8 = two proven ne=4 groups (sub∈{0,4}). q6_0's ++ // qh is BYTE-INDEXED per lane (no packed-int read), so there is no overread; each group is byte-identical ++ // to the ne<=4 body at is=i0+sub. See q4_1/q5_0/q5_1. ++#pragma unroll ++ for (int sub = 0; sub < ne; sub += (ne <= 4 ? ne : 4)) { ++ const int64_t is = i0 + sub; ++ const int64_t ib = is / QK6_0; ++ const int idq = is % QK6_0; ++ const int iqs = is % (QK6_0/2); ++ const int shift = (is % QK6_0) / (QK6_0/2); ++ constexpr int nl = ne <= 4 ? ne : 4; ++ ++ int q; ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; + +- { +- // High 2 bits, ported verbatim from ik_llama.cpp dequantize_1_q6_0 (fattn-common.cuh:577-578) +- // and evaluated per output lane l at value index (idq + l). For value V: byte = qh[V%(QK6_0/4)], +- // 2-bit field at shift = 4*((V/(QK6_0/4))%2) + 2*shift. `shift` is constant across the ne lanes +- // of one call (i0 is aligned within a nibble-half), so it equals (idq+l)/(QK6_0/2) for every l. +- // Verified byte-exact against the host quantize_row_q6_0 pack for all 32 block positions. ++ { ++ // High 2 bits, ported verbatim from ik_llama.cpp dequantize_1_q6_0 (fattn-common.cuh:577-578) ++ // and evaluated per output lane l at value index (idq + l). For value V: byte = qh[V%(QK6_0/4)], ++ // 2-bit field at shift = 4*((V/(QK6_0/4))%2) + 2*shift. `shift` is constant across the lanes ++ // of one group (is is aligned within a nibble-half), so it equals (idq+l)/(QK6_0/2) for every l. ++ // Verified byte-exact against the host quantize_row_q6_0 pack for all 32 block positions. + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- const int j = idq + l; +- const int h = (int) x[ib].qh[j % (QK6_0/4)] >> (4*((j / (QK6_0/4)) % 2) + 2*shift); +- q |= (h & 0x03) << (8*l + 4); ++ for (int l = 0; l < nl; ++l) { ++ const int j = idq + l; ++ const int h = (int) x[ib].qh[j % (QK6_0/4)] >> (4*((j / (QK6_0/4)) % 2) + 2*shift); ++ q |= (h & 0x03) << (8*l + 4); ++ } + } +- } + +- q = __vsubss4(q, 0x20202020); // q6_0 level offset is 32 (0x20), vs q5_0's 16 ++ q = __vsubss4(q, 0x20202020); // q6_0 level offset is 32 (0x20), vs q5_0's 16 + +- const int8_t * q8 = (const int8_t *) &q; ++ const int8_t * q8 = (const int8_t *) &q; + + #ifdef FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const half2 d = __half2half2(x[ib].d); ++ if constexpr (std::is_same_v) { ++ const half2 d = __half2half2(x[ib].d); + + #pragma unroll +- for (int l0 = 0; l0 < ne; l0 += 2) { +- ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); +- } +- } else ++ for (int l0 = 0; l0 < nl; l0 += 2) { ++ ((half2 *) dst)[(sub + l0)/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]); ++ } ++ } else + #endif // FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const float d = x[ib].d; ++ if constexpr (std::is_same_v) { ++ const float d = x[ib].d; + + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- ((float *) dst)[l] = d * q8[l]; ++ for (int l = 0; l < nl; ++l) { ++ ((float *) dst)[sub + l] = d * q8[l]; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); + } +- } else { +- static_assert(std::is_same_v, "bad type"); + } + } + +@@ -638,49 +704,56 @@ template + static __device__ __forceinline__ void dequantize_V_q5_1(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) { + const block_q5_1 * x = (const block_q5_1 *) vx; + +- const int64_t ib = i0 / QK5_1; +- const int idq = i0 % QK5_1; +- const int iqs = i0 % (QK5_1/2); +- const int shift = (i0 % QK5_1) / (QK5_1/2); +- +- int q; +- static_assert(ne == 2 || ne == 4, "bad ne"); +- ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +- q >>= 4*shift; +- q &= 0x0F0F0F0F; ++ static_assert(ne == 2 || ne == 4 || ne == 8, "bad ne"); ++ // opencoti-hook: P0 DCA-on scalar in-register (#644) — ne==8 = two proven ne=4 groups (sub∈{0,4}); the ++ // full 4-byte qh is re-read per group (ggml_cuda_memcpy_1, nl=4), no overread, idq+l ≤31. See q4_1/q5_0. ++#pragma unroll ++ for (int sub = 0; sub < ne; sub += (ne <= 4 ? ne : 4)) { ++ const int64_t is = i0 + sub; ++ const int64_t ib = is / QK5_1; ++ const int idq = is % QK5_1; ++ const int iqs = is % (QK5_1/2); ++ const int shift = (is % QK5_1) / (QK5_1/2); ++ constexpr int nl = ne <= 4 ? ne : 4; ++ ++ int q; ++ ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); ++ q >>= 4*shift; ++ q &= 0x0F0F0F0F; + +- { +- int qh; +- ggml_cuda_memcpy_1(&qh, x[ib].qh); ++ { ++ int qh; ++ ggml_cuda_memcpy_1(&qh, x[ib].qh); + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4); ++ for (int l = 0; l < nl; ++l) { ++ q |= ((qh >> (idq + l)) & 0x00000001) << (8*l + 4); ++ } + } +- } + +- const int8_t * q8 = (const int8_t *) &q; ++ const int8_t * q8 = (const int8_t *) &q; + + #ifdef FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const half2 dm = x[ib].dm; +- const half2 d = __half2half2( __low2half(dm)); +- const half2 m = __half2half2(__high2half(dm)); ++ if constexpr (std::is_same_v) { ++ const half2 dm = x[ib].dm; ++ const half2 d = __half2half2( __low2half(dm)); ++ const half2 m = __half2half2(__high2half(dm)); + + #pragma unroll +- for (int l0 = 0; l0 < ne; l0 += 2) { +- ((half2 *) dst)[l0/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; +- } +- } else ++ for (int l0 = 0; l0 < nl; l0 += 2) { ++ ((half2 *) dst)[(sub + l0)/2] = d * make_half2(q8[l0 + 0], q8[l0 + 1]) + m; ++ } ++ } else + #endif // FP16_AVAILABLE +- if constexpr (std::is_same_v) { +- const float2 dm = __half22float2(x[ib].dm); ++ if constexpr (std::is_same_v) { ++ const float2 dm = __half22float2(x[ib].dm); + + #pragma unroll +- for (int l = 0; l < ne; ++l) { +- ((float *) dst)[l] = dm.x * q8[l] + dm.y; ++ for (int l = 0; l < nl; ++l) { ++ ((float *) dst)[sub + l] = dm.x * q8[l] + dm.y; ++ } ++ } else { ++ static_assert(std::is_same_v, "bad type"); + } +- } else { +- static_assert(std::is_same_v, "bad type"); + } + } + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +index 9d151be..a91c6aa 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +@@ -2566,9 +2566,13 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + // get_dequantize_V and never needs to_fp16_nc (TCQ has no to_fp16_nc — it lifts via cpy_turbo*_tcq_f16), + // so exempt it from the converter requirement. K is always materialized when quantized, so it keeps the + // converter requirement unconditionally. +- GGML_ASSERT((K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) && ++ // opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — when the kernel is instantiated for a ++ // scalar-quant type_K (q8_0/q4_0/…) it reads the raw quant K in-register (load_tile_dequant / ++ // get_dequantize_V, same generic path turbo uses), so it never needs a to_fp16_nc converter — exempt it, ++ // mirroring the type_V clause. type_K==F16 keeps the shipped in-launch materialize (converter required). ++ GGML_ASSERT((type_K != GGML_TYPE_F16 || K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) && + (type_V != GGML_TYPE_F16 || V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) && +- "dca fused: K must be f16 or have a to_fp16_nc converter; a materialized (type_V==F16) V likewise"); ++ "dca fused: a materialized (type==F16) K/V must be f16 or have a to_fp16_nc converter"); + GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors"); + // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER + // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent. +@@ -2631,7 +2635,12 @@ void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case(ggml_backend_cuda_context & + size_t nb_k1 = K->nb[1], nb_k2 = K->nb[2], nb_k3 = K->nb[3]; + const char * v_data = (const char *) V->data; + size_t nb_v1 = V->nb[1], nb_v2 = V->nb[2], nb_v3 = V->nb[3]; +- if (K->type != GGML_TYPE_F16) { ++ // opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — materialize K to f16 ONLY when the ++ // kernel is instantiated F16 (the shipped in-launch lift). A scalar-typed instance (type_K != F16) reads ++ // the raw quant K in-register via load_tile_dequant / get_dequantize_V (killing the whole-cache ++ // to_fp16_nc cast that dominated q8K/q4V DCA-on decode — ~21.9s of the 43.4s lift, #582-P0 profile), so ++ // it leaves k_data/nb_k* on the raw pointer. Mirrors the type_V==F16 guard on the V block below. ++ if (type_K == GGML_TYPE_F16 && K->type != GGML_TYPE_F16) { + const size_t ts = ggml_type_size(K->type); + GGML_ASSERT(K->nb[0] == ts && "dca fused: quant K must have contiguous dim-0 (block axis)"); + to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(K->type); +@@ -2954,3 +2963,47 @@ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 1, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 2, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 4, 8) + EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_D256( 8, 8) ++ ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — SCALAR-quant K/V read in-register in the ++// dca_fused MMA kernel, killing the whole-cache to_fp16_nc dequant-lift that dominated q8K/q4V DCA-on decode ++// (#582-P0 profile: ~29.5s/decode = 43.4s prefill+decode of dequantize_block; FA kernel itself is already ++// f16-identical to the f16-KV cell). Reuses the SAME generic load_tile_dequant / get_dequantize_V path turbo ++// uses (get_dequantize_V already routes q8_0/q4_0, fattn-common.cuh:764/778/788) — no new kernel math. Scalar ++// KV is NOT FWHT-rotated ⇒ NO output wht_o (the one place scalar diverges from the turbo path). Beachhead: ++// D128 (Qwen2.5-14B-1M full-attention, the P0 gate model), K=q8_0 / V=q4_0 (the HAL recommended hi-K/cheap-V ++// recipe). #644 GENERALIZES to the full usable scalar matrix at D128: the symmetric diagonal ++// {q4_0,q4_1,q5_0,q5_1,q6_0,q8_0} K==V + the hi-K/cheap-V recipes (q8K/{q4,q5,q6}V, q6K/q4V). The 5/6-bit V ++// readers gained an ne=8 path (fattn-common.cuh, #644). D256/D512 scalar deferred (Gemma's KV story is turbo, ++// already in-register there). Decode-gated in dca.cpp (n_q ≤ WS2_NQ_MAX); large-n_q prefill keeps the f16-lift ++// path (byte-identical to shipped). One extern per (ncols1,ncols2,type_K,type_V). ++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(ncols1, ncols2, type_K, type_V) \ ++ extern DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, ncols1, ncols2, type_K, type_V); ++ ++#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 8, 1, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(16, 1, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(32, 1, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(64, 1, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 4, 2, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 8, 2, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(16, 2, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(32, 2, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 2, 4, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 4, 4, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 8, 4, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128(16, 4, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 1, 8, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 2, 8, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 4, 8, type_K, type_V) \ ++ EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128( 8, 8, type_K, type_V) ++ ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) // beachhead #643 ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) // #644 symmetric ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q6_0, GGML_TYPE_Q6_0) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) // #644 hi-K/cheap-V ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q8_0, GGML_TYPE_Q6_0) ++EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_SCALAR_D128_ALL16(GGML_TYPE_Q6_0, GGML_TYPE_Q4_0) +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +index 29f1a1e..e76054f 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -867,11 +867,43 @@ static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa(ggml_backend + // only ever uses DKQ==DV (key_length==value_length on every supported arch), so dispatch on Q->ne[0] alone. + static void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; ++ // opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — per-op n_q gate (mirrors the WS2 turbo-V ++ // dca.cpp gate, but here because a scalar q8_0/q4_0 cache is deferred RAW at BOTH prefill and decode ++ // (bug-720: the in-launch whole-KV materialize is the fast PREFILL path, so we must NOT lift-to-f16 in the ++ // graph like turbo does). Q->ne[1] is the per-ubatch n_q on the fused op's permuted Q ([dh, n_q, n_head, 1]). ++ // Decode/small-verify (n_q ≤ WS2_NQ_MAX) → in-register scalar instance (kills the ~29.5s/decode whole-KV ++ // dequant-lift, HAL §3b); prefill (large n_q) → F16/F16 default instance = the shipped in-launch whole-KV ++ // materialize (in-register loses ~8-17% at prefill, #629). Both are byte-identical in output. ++ static const bool ws2_dca_scalar_kv = []() { const char * e = getenv("WS2_DCA_SCALAR_KV"); return (!e || e[0] != '0'); }(); ++ static const int ws2_nq_max = []() { const char * e = getenv("WS2_NQ_MAX"); return e ? atoi(e) : 16; }(); ++ const bool scalar_inreg = ws2_dca_scalar_kv && Q->ne[1] <= ws2_nq_max; + switch (Q->ne[0]) { + case 128: + GGML_ASSERT(V->ne[0] == 128); +- ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<128>(ctx, dst); break; ++ // opencoti-hook: P0 DCA-on scalar in-register (#643 beachhead + #644 full matrix) — the D128 ++ // full-attention case (Qwen2.5-14B-1M etc.). Built in-register scalar pairs = the symmetric diagonal ++ // {q4_0,q4_1,q5_0,q5_1,q6_0,q8_0} K==V + hi-K/cheap-V recipes (q8K/{q4,q5,q6}V, q6K/q4V). A matched ++ // pair (decode, scalar_inreg) dispatches its typed instance; any other combo (incl. non-decode, or a ++ // pair we did not build) falls through to the default F16-materialize path (correct, the shipped tax). ++ if (scalar_inreg) { ++#define OCTI_DCA_SCALAR_D128(KT, VT) \ ++ if (K->type == (KT) && V->type == (VT)) { ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<128, KT, VT>(ctx, dst); break; } ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) // beachhead #643 ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) // #644 symmetric diagonal ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q6_0, GGML_TYPE_Q6_0) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) // #644 hi-K/cheap-V ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q8_0, GGML_TYPE_Q6_0) ++ OCTI_DCA_SCALAR_D128(GGML_TYPE_Q6_0, GGML_TYPE_Q4_0) ++#undef OCTI_DCA_SCALAR_D128 ++ } ++ ggml_cuda_flash_attn_ext_mma_f16_dca_fused_dispatch_gqa<128>(ctx, dst); ++ break; + case 256: + GGML_ASSERT(V->ne[0] == 256); + // opencoti-hook: WS2 turbo-V DCA (#629, S3c) — D256 GLOBAL head_dim is the Qwen-arch case +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-0.cu +new file mode 100644 +index 0000000..a6e6eec +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_1 / V=Q4_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_1/Q4_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-1.cu +new file mode 100644 +index 0000000..7cb95c3 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_1 / V=Q4_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_1/Q4_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-2.cu +new file mode 100644 +index 0000000..aea5a5e +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_1 / V=Q4_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_1/Q4_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-3.cu +new file mode 100644 +index 0000000..beb5507 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q41q41-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_1 / V=Q4_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_1/Q4_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-0.cu +new file mode 100644 +index 0000000..5d8c0f0 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-1.cu +new file mode 100644 +index 0000000..6dd841f +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-2.cu +new file mode 100644 +index 0000000..bebb71a +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-3.cu +new file mode 100644 +index 0000000..8ec1f1a +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q4q4-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q4_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q4_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-0.cu +new file mode 100644 +index 0000000..1d891f4 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_1 / V=Q5_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_1/Q5_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-1.cu +new file mode 100644 +index 0000000..c96440f +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_1 / V=Q5_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_1/Q5_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-2.cu +new file mode 100644 +index 0000000..9aaba59 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_1 / V=Q5_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_1/Q5_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-3.cu +new file mode 100644 +index 0000000..7143ac0 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q51q51-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_1 / V=Q5_1 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_1/Q5_1). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-0.cu +new file mode 100644 +index 0000000..28572e6 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-1.cu +new file mode 100644 +index 0000000..e29c140 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-2.cu +new file mode 100644 +index 0000000..6814a8e +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-3.cu +new file mode 100644 +index 0000000..3faa647 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q5q5-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q5_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q5_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-0.cu +new file mode 100644 +index 0000000..cb1da1e +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-1.cu +new file mode 100644 +index 0000000..33aaa62 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-2.cu +new file mode 100644 +index 0000000..f4e4933 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-3.cu +new file mode 100644 +index 0000000..c78e3c5 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q4-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q4_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q4_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-0.cu +new file mode 100644 +index 0000000..d9cfff5 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-1.cu +new file mode 100644 +index 0000000..149db78 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-2.cu +new file mode 100644 +index 0000000..66d63b9 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-3.cu +new file mode 100644 +index 0000000..bae4f48 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q6q6-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q6_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q6_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-0.cu +new file mode 100644 +index 0000000..de9cf93 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — D128 (Qwen2.5-14B-1M full-attention) ++// SCALAR K=q8_0 / V=q4_0 in-register multichunk DCA fused instances (reuses the type-parametrized case; ++// get_dequantize_V already routes q8_0/q4_0). Sharded 4 configs/TU (matches the turbo sharding, #531). ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-1.cu +new file mode 100644 +index 0000000..c1633e1 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — D128 (Qwen2.5-14B-1M full-attention) ++// SCALAR K=q8_0 / V=q4_0 in-register multichunk DCA fused instances (reuses the type-parametrized case; ++// get_dequantize_V already routes q8_0/q4_0). Sharded 4 configs/TU (matches the turbo sharding, #531). ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-2.cu +new file mode 100644 +index 0000000..b1d506f +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — D128 (Qwen2.5-14B-1M full-attention) ++// SCALAR K=q8_0 / V=q4_0 in-register multichunk DCA fused instances (reuses the type-parametrized case; ++// get_dequantize_V already routes q8_0/q4_0). Sharded 4 configs/TU (matches the turbo sharding, #531). ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-3.cu +new file mode 100644 +index 0000000..0077837 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q4-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#643, #582-P0) — D128 (Qwen2.5-14B-1M full-attention) ++// SCALAR K=q8_0 / V=q4_0 in-register multichunk DCA fused instances (reuses the type-parametrized case; ++// get_dequantize_V already routes q8_0/q4_0). Sharded 4 configs/TU (matches the turbo sharding, #531). ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-0.cu +new file mode 100644 +index 0000000..e4a2c1d +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-1.cu +new file mode 100644 +index 0000000..f0d0770 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-2.cu +new file mode 100644 +index 0000000..46f04f4 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-3.cu +new file mode 100644 +index 0000000..c752889 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q5-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q5_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q5_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-0.cu +new file mode 100644 +index 0000000..b809d1b +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-1.cu +new file mode 100644 +index 0000000..6cd0195 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-2.cu +new file mode 100644 +index 0000000..4a29273 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-3.cu +new file mode 100644 +index 0000000..c872d91 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q6-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q6_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q6_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-0.cu +new file mode 100644 +index 0000000..6ce5aae +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-0.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q8_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q8_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 64, 1, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-1.cu +new file mode 100644 +index 0000000..b4e3f99 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-1.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q8_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q8_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 32, 2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-2.cu +new file mode 100644 +index 0000000..dbf193e +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-2.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q8_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q8_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 16, 4, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-3.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-3.cu +new file mode 100644 +index 0000000..b85977a +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-scalar-q8q8-d128-instance-3.cu +@@ -0,0 +1,9 @@ ++// opencoti-hook: P0 DCA-on scalar in-register (#644, #582-P0) — D128 SCALAR K=Q8_0 / V=Q8_0 ++// in-register multichunk DCA fused instances (type-parametrized case; get_dequantize_V + the ne=8 ++// V-readers route Q8_0/Q8_0). Decode-only path (prefill uses the F16 materialize default). Sharded 4/TU. ++#include "../fattn-mma-f16.cuh" ++ ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 1, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 2, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 4, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); ++DECL_FATTN_MMA_F16_DCA_FUSED_TURBO_CASE(128, 128, 8, 8, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); diff --git a/patches/0120-p2-mixed-kv.patch b/patches/0120-p2-mixed-kv.patch new file mode 100644 index 0000000000000000000000000000000000000000..04abd6b9fa37864b1894d107bf8fef260539fe04 --- /dev/null +++ b/patches/0120-p2-mixed-kv.patch @@ -0,0 +1,601 @@ +opencoti patch 0120 — P2 position-axis mixed KV: f16/high-bit resident window ⊕ compressed VRAM/host tail (#622 / #582-P2) + +Adds a per-position KV-precision split on top of the rolling-KV position window +(#585/M7 Stage 3c): the resident window [0,wc) keeps the boot-selected KV type +(e.g. q8_0), while the spilled tail [wc,n_kv) may carry a MORE-compressed type +(e.g. q4_0). Recent tokens — which dominate attention — stay high-fidelity; the +older tail pays fewer H2D bytes. This is the position-axis complement to the +head-axis (#620/#643 P0) and boot-tier (#621 P1) work in the #582 line. + +CLI: `-ctkt ` / `-ctvt ` set the tail K/V type. Sentinel +GGML_TYPE_COUNT (the default when the flag is unset) = "tail same as window", in +which case every path below is byte-identical to the uniform-window behaviour. + +Plumbing (5-point, additive): common CLI (`common/arg.cpp`) → +`common_params.cache_type_k_tail/_v_tail` (`common/common.h`,`common.cpp`) → +public `llama_context_params` (`include/llama.h`) → `llama_cparams` +(`src/llama-cparams.h`) → the kv-cache ctor (`src/llama-kv-cache*.{cpp,h}`, +`src/llama-context.cpp`, `src/llama-model.cpp`), which allocates the pinned host +tail (`k_cpu_per_stream`/`v_cpu_per_stream`) at the tail type when it differs. + +Decode path (the win): the POSITION_WINDOW streaming FA op +(`ggml/src/ggml-cuda/fattn.cu`) reads the window and the tail with their own +(possibly different) types in-register — no whole-cache f16 materialise — reusing +the #620/#629/#619 in-register dequant readers. Mixed q8-window/q4-tail streams +half the tail bytes of a uniform-q8 tail. + +Correctness-fallback paths made mixed-type-safe (host, `src/llama-kv-cache.cpp`): + * bug-2161 — the POSITION_WINDOW get_k/get_v concat reassembly accessor + (prefill / graph-reserve, large n_q) hit `ggml_concat`'s a->type==b->type + assert on a mixed window⊕tail. Fixed with a file-local `poswin_lift_to_f16` + (exact mirror of dca.cpp:dca_lift_to_f16 — f16 passthrough; scalar quants + lift via an f32 ggml_cast intermediate then f16, both legs GPU CPY, byte- + exact to a single-pass dequant) applied to BOTH region views only when the + types differ. + * bug-2162 — the Stage 3c-6 state-IO window paths (K/V × write/read) used the + window row size for the tail too, so an in-memory context-checkpoint save/ + restore (PR16391, on by default) between prompts asserted on the 2nd prompt. + Fixed: the tail region uses ggml_row_size(k_cpu->type/v_cpu->type,…) for its + own offsets/lengths; symmetric writer/reader; parity assert recomputed as + window_cells*window_row + tail_cells*tail_row. Byte-identical when the tail + type == the window type (tail_row == window_row). + +Gate (S7, bs2 RTX 6000, Qwen2.5-14B-Instruct-1M-Q8_0, ctx 24576, 6 prompts, +logit-equiv vs the model's own full-resident run = gold; NEVER greedy needle): + * CORRECTNESS: real_frac=0 for uniform-q8 spill (U8), mixed (MIX), uniform-q4 + (U4) — zero real top-1 disagreements at needle-answer scope. Advisory mean-TV + U8 0.0069 < MIX 0.0110 < U4 0.0118 — mixed sits strictly between uniform-q8 + (gold) and uniform-q4, closer to gold than uniform-q4. + * THROUGHPUT: MIX 28.1 tps vs U8 26.4 = +6.4% (q4 tail = half the tail H2D + bytes); tail 11264 cells engaged. + * REGRESSION: `-ctkt q8_0 -ctvt q8_0` (explicit tail==window sentinel) is + byte-identical to unset (U8) — the mixed plumbing is inert off-path. + +Additive soft-fork; opencoti-hook markers registered in UPSTREAM_SYNC.md. +Byte-identical to the shipped behaviour whenever no distinct tail type is set. + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -2323,6 +2323,35 @@ + params.cache_type_v = kv_cache_type_from_str(value); + } + ).set_env("LLAMA_ARG_CACHE_TYPE_V")); ++ // opencoti #582-P2 (position-axis mixed KV) — spilled-tail KV type. Only meaningful in ++ // POSITION_WINDOW residency mode: the resident recent window keeps -ctk/-ctv; older ++ // cells that spill to the host tail are stored at this (typically more-compressed) type. ++ // The read path is dequant-on-lift, so any supported KV type composes. See ++ // docs/evaluations/hal_kv_compression.md. ++ add_opt(common_arg( ++ {"-ctkt", "--cache-type-k-tail"}, "TYPE", ++ string_format( ++ "opencoti #582-P2: KV cache data type for the spilled position TAIL K\n" ++ "(POSITION_WINDOW mode only; default: same as -ctk)\n" ++ "allowed values: %s", ++ get_all_kv_cache_types().c_str() ++ ), ++ [](common_params & params, const std::string & value) { ++ params.cache_type_k_tail = kv_cache_type_from_str(value); ++ } ++ ).set_env("LLAMA_ARG_CACHE_TYPE_K_TAIL")); ++ add_opt(common_arg( ++ {"-ctvt", "--cache-type-v-tail"}, "TYPE", ++ string_format( ++ "opencoti #582-P2: KV cache data type for the spilled position TAIL V\n" ++ "(POSITION_WINDOW mode only; default: same as -ctv)\n" ++ "allowed values: %s", ++ get_all_kv_cache_types().c_str() ++ ), ++ [](common_params & params, const std::string & value) { ++ params.cache_type_v_tail = kv_cache_type_from_str(value); ++ } ++ ).set_env("LLAMA_ARG_CACHE_TYPE_V_TAIL")); + add_opt(common_arg( + {"--hellaswag"}, + "compute HellaSwag score over random tasks from datafile supplied with -f", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1685,6 +1685,9 @@ + + cparams.type_k = params.cache_type_k; + cparams.type_v = params.cache_type_v; ++ // opencoti #582-P2 (position-axis mixed KV) — distinct spilled-tail element type. ++ cparams.type_k_tail = params.cache_type_k_tail; ++ cparams.type_v_tail = params.cache_type_v_tail; + + return cparams; + } +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -640,6 +640,11 @@ + + ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K + ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V ++ // opencoti #582-P2 (position-axis mixed KV) — element type for the spilled ++ // position TAIL vs the resident window (cache_type_k/v). GGML_TYPE_COUNT = ++ // same as window (uniform). See docs/evaluations/hal_kv_compression.md. ++ ggml_type cache_type_k_tail = GGML_TYPE_COUNT; ++ ggml_type cache_type_v_tail = GGML_TYPE_COUNT; + + common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO; + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -1257,7 +1257,10 @@ + const bool two_region = (K_tail != nullptr); + if (two_region) { + GGML_ASSERT(V_tail != nullptr); +- GGML_ASSERT(K_tail->type == K->type && V_tail->type == V->type); ++ // opencoti #582-P2 (position-axis mixed KV): the spilled tail MAY carry a ++ // distinct (more-compressed) element type than the resident window. Both ++ // regions are dequant-on-lift into an f16 slot, so per-region converters ++ // (below) handle any type mix — only the geometry must match. + GGML_ASSERT(K_tail->ne[0] == K->ne[0] && V_tail->ne[0] == V->ne[0]); + GGML_ASSERT(K_tail->ne[2] == K->ne[2]); + } +@@ -1272,10 +1275,24 @@ + // resident; (2) a quantized type that ends up with no slot to dequant into + // (n_slots==0) stays resident — both enforced below. The f16 path resolves + // to nullptr converters and is byte-identical to pre-S3d. +- const bool all_f16 = (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16); +- to_fp16_nc_cuda_t to_fp16_k = all_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(K->type); +- to_fp16_nc_cuda_t to_fp16_v = all_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(V->type); +- if (!all_f16 && (to_fp16_k == nullptr || to_fp16_v == nullptr)) { ++ // opencoti #582-P2 (position-axis mixed KV): the window and the tail may carry ++ // DISTINCT element types (e.g. f16 recent window ⊕ q4_0 spilled tail). Each ++ // region dequant-on-lifts through its OWN converter; the memcpy fast path is ++ // taken only for an f16 region. `all_f16` (both regions f16) still gates the ++ // slot-forcing / short-circuit guards below so any quant region forces slots. ++ // Note: ggml_get_to_fp16_nc_cuda has NO F16 case (returns nullptr), so an f16 ++ // region MUST use the memcpy path — never the converter. For uniform configs ++ // (single region, or tail==window type) this is byte-identical to pre-P2. ++ const bool win_f16 = (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16); ++ const bool tail_f16 = !two_region || ++ (K_tail->type == GGML_TYPE_F16 && V_tail->type == GGML_TYPE_F16); ++ const bool all_f16 = win_f16 && tail_f16; ++ to_fp16_nc_cuda_t to_fp16_k = win_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(K->type); ++ to_fp16_nc_cuda_t to_fp16_v = win_f16 ? nullptr : ggml_get_to_fp16_nc_cuda(V->type); ++ to_fp16_nc_cuda_t to_fp16_k_tail = (two_region && !tail_f16) ? ggml_get_to_fp16_nc_cuda(K_tail->type) : nullptr; ++ to_fp16_nc_cuda_t to_fp16_v_tail = (two_region && !tail_f16) ? ggml_get_to_fp16_nc_cuda(V_tail->type) : nullptr; ++ if ((!win_f16 && (to_fp16_k == nullptr || to_fp16_v == nullptr)) || ++ (two_region && !tail_f16 && (to_fp16_k_tail == nullptr || to_fp16_v_tail == nullptr))) { + ggml_cuda_flash_attn_ext(ctx, dst); // unsupported quant type → resident + return; + } +@@ -1473,6 +1490,11 @@ + // all_f16 fast path (the memcpy lift) but cheap to compute. + const size_t ts_k = ggml_type_size(K->type); + const size_t ts_v = ggml_type_size(V->type); ++ // opencoti #582-P2 — the spilled tail may be a distinct type; its converter ++ // strides use the tail's own type-size (s = nb/ts). Same as ts_k/ts_v for ++ // uniform configs (single region or tail==window type). ++ const size_t ts_k_tail = two_region ? ggml_type_size(K_tail->type) : ts_k; ++ const size_t ts_v_tail = two_region ? ggml_type_size(V_tail->type) : ts_v; + // opencoti-hook: f5-rolling-kv #586 — PERSISTENT staging ring (was + // ggml_cuda_pool_alloc). The pool re-handed the SAME slot addresses every + // op/layer, which is the sole reason the coarse per-op cs_sync barrier +@@ -1599,12 +1621,24 @@ + const bool use_2d = two_region + ? ((size_t) tK->nb[1] != k_row || (size_t) tV->nb[1] != v_row) + : tr.host; ++ // opencoti #582-P2: select this tile's converter/type-size by region. ++ // A tail tile (source == K_tail) uses the tail's own type; a window ++ // tile uses the window's. An f16 region (reg_f16) takes the memcpy ++ // fast path; a quant region takes its region-keyed converter. For ++ // uniform configs reg_f16==all_f16 and the region vars alias the ++ // window ones → byte-identical. ++ const bool tile_is_tail = two_region && (tK == K_tail); ++ const bool reg_f16 = tile_is_tail ? tail_f16 : win_f16; ++ const to_fp16_nc_cuda_t rk = tile_is_tail ? to_fp16_k_tail : to_fp16_k; ++ const to_fp16_nc_cuda_t rv = tile_is_tail ? to_fp16_v_tail : to_fp16_v; ++ const size_t rts_k = tile_is_tail ? ts_k_tail : ts_k; ++ const size_t rts_v = tile_is_tail ? ts_v_tail : ts_v; + const int slot = t % n_slots; + const size_t k_nb2 = (size_t)this_kv * k_row; + const size_t v_nb2 = (size_t)this_kv * v_row; + char * sK = slotK_base + (size_t)slot * k_slot; + char * sV = slotV_base + (size_t)slot * v_slot; +- if (!all_f16) { ++ if (!reg_f16) { + // S3d dequant-on-lift: ONE nc-converter launch per tile reads the + // (head_dim × this_kv × n_head_kv × n_b) source sub-block at its + // native strides (s = nb/ts: blocks-per-row for quant, elements +@@ -1638,12 +1672,12 @@ + qsrcK = qscrK; + qsrcV = qscrV; + } +- to_fp16_k(qsrcK, (half *)sK, ++ rk(qsrcK, (half *)sK, + head_dim, this_kv, n_head_kv, n_b, +- (int64_t)(tK->nb[1]/ts_k), (int64_t)(tK->nb[2]/ts_k), (int64_t)(tK->nb[3]/ts_k), cs); +- to_fp16_v(qsrcV, (half *)sV, ++ (int64_t)(tK->nb[1]/rts_k), (int64_t)(tK->nb[2]/rts_k), (int64_t)(tK->nb[3]/rts_k), cs); ++ rv(qsrcV, (half *)sV, + dv, this_kv, n_head_kv, n_b, +- (int64_t)(tV->nb[1]/ts_v), (int64_t)(tV->nb[2]/ts_v), (int64_t)(tV->nb[3]/ts_v), cs); ++ (int64_t)(tV->nb[1]/rts_v), (int64_t)(tV->nb[2]/rts_v), (int64_t)(tV->nb[3]/rts_v), cs); + } else + for (int bb = 0; bb < n_b; ++bb) { + // opencoti-hook: f5-rolling-kv #586 — bulk H2D + on-device repack. +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -369,6 +369,12 @@ + // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head + // (force M2 split), 2 = window (force POSITION_WINDOW, warn+fallback if ineligible). + uint32_t kv_residency_mode; ++ // opencoti #582-P2 (position-axis mixed KV) — element type for the position ++ // TAIL vs the resident window (type_k/type_v). GGML_TYPE_COUNT = same as ++ // window (uniform). Consulted only in POSITION_WINDOW mode; dequant-on-lift ++ // read path means any supported KV type composes. ++ enum ggml_type type_k_tail; ++ enum ggml_type type_v_tail; + // opencoti F5 M3 neo — see docs/features/advanced_kv.md + // 0 = off (default), 1 = on, 2 = auto. Effective only when the + // M2 head split is active for a given layer. +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -106,6 +106,9 @@ + cparams.pcie_bw_gbps = params.pcie_bw_gbps; + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md + cparams.kv_residency_mode = params.kv_residency_mode; ++ // opencoti #582-P2 (position-axis mixed KV) — distinct tail element type. ++ cparams.type_k_tail = params.type_k_tail; ++ cparams.type_v_tail = params.type_v_tail; + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). INTERNAL: + // the memory init below runs a measurement pass, then re-creates with the measured + // reserve; these start at the "pass 1 / default reserve" values. See +@@ -3744,6 +3747,8 @@ + /*.vram_target_mib =*/ 0, // opencoti F5 M7 Rolling KV — 0 = maximize + /*.pcie_bw_gbps =*/ 0.0f, // opencoti F5 M7 Rolling KV — 0 = unknown + /*.kv_residency_mode =*/ 0, // opencoti F5 M7 Rolling KV — 0 = auto ++ /*.type_k_tail =*/ GGML_TYPE_COUNT, // opencoti #582-P2 — same as window ++ /*.type_v_tail =*/ GGML_TYPE_COUNT, // opencoti #582-P2 — same as window + /*.neo_pipeline_mode =*/ 0, // opencoti F5 M3 neo — off by default + /*.fused_moe_up_gate =*/ false, // opencoti F5 W3 #292 — off by default + /*.dca_enabled =*/ false, // opencoti F5 dca — off by default +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -35,6 +35,14 @@ + // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. + // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head, 2 = window. + uint32_t kv_residency_mode; ++ // opencoti #582-P2 (position-axis mixed KV) — distinct element type for the ++ // position TAIL (spilled/older cells) vs the resident window (type_k/type_v). ++ // GGML_TYPE_COUNT = "same as window" (byte-identical uniform storage). A more- ++ // compressed tail (e.g. window q8_0 ⊕ tail q4_0) shrinks the streamed overflow. ++ // Consulted only in POSITION_WINDOW mode; the read path is dequant-on-lift, so ++ // any supported KV type composes. See docs/evaluations/hal_kv_compression.md. ++ ggml_type type_k_tail; ++ ggml_type type_v_tail; + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) — + // see docs/features/rolling_kv_compute_reserve.md. INTERNAL (not user params): + // set by the llama_context two-pass measurement, NOT from llama_context_params. +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -639,7 +639,9 @@ + const layer_reuse_cb & reuse, + llama_memory_t mem_other, + const layer_share_cb & share, +- uint32_t sparse_attn_block_size) : ++ uint32_t sparse_attn_block_size, ++ ggml_type type_k_tail, ++ ggml_type type_v_tail) : + model(model), hparams(model.hparams), v_trans(v_trans), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type), + // opencoti bug-858 dual-context MTP — `other` is the TARGET cache (nullptr for every +@@ -1123,11 +1125,18 @@ + const uint32_t host_k_embd = layer_window ? n_embd_k_gqa : n_embd_k_cpu; + const uint32_t host_v_embd = layer_window ? n_embd_v_gqa : n_embd_v_cpu; + const uint32_t host_cells = layer_window ? tail_c : kv_size; ++ // opencoti #582-P2 (mixed-type KV): the position-tail may store a ++ // MORE-compressed type than the resident window. `type_k_tail` / ++ // `type_v_tail` default to GGML_TYPE_COUNT (== "same as window"), ++ // so this is byte-identical until a mode selects a distinct tail ++ // type. The M2 head-half (gpu_heads path) always keeps type_k/v. ++ const ggml_type ttk = (layer_window && type_k_tail != GGML_TYPE_COUNT) ? type_k_tail : type_k; ++ const ggml_type ttv = (layer_window && type_v_tail != GGML_TYPE_COUNT) ? type_v_tail : type_v; + ggml_tensor * k_cpu_s = has_k +- ? ggml_new_tensor_3d(ctx_cpu_s, type_k, host_k_embd, host_cells, 1) ++ ? ggml_new_tensor_3d(ctx_cpu_s, ttk, host_k_embd, host_cells, 1) + : nullptr; + ggml_tensor * v_cpu_s = has_v +- ? ggml_new_tensor_3d(ctx_cpu_s, type_v, host_v_embd, host_cells, 1) ++ ? ggml_new_tensor_3d(ctx_cpu_s, ttv, host_v_embd, host_cells, 1) + : nullptr; + // Name distinguishes the two layouts for state-IO grep: + // "cache_k_tail" (position tail) vs "cache_k_cpu" (head half). +@@ -2784,6 +2793,26 @@ + return layers[0].v_per_stream[0]->type; + } + ++// opencoti #582-P2 (mixed-type KV): the position-tail's element type. In ++// window mode the tail lives in k_cpu_per_stream/v_cpu_per_stream (ALL heads, ++// possibly a more-compressed type than the resident window); read it there. ++// Outside window mode (or M2 head-split, where the host half mirrors the ++// window type) there is no distinct tail — fall back to the window type so the ++// accessor is byte-identical for every non-P2 config. ++ggml_type llama_kv_cache::type_k_tail() const { ++ if (!layers[0].k_cpu_per_stream.empty() && layers[0].k_cpu_per_stream[0]) { ++ return layers[0].k_cpu_per_stream[0]->type; ++ } ++ return type_k(); ++} ++ ++ggml_type llama_kv_cache::type_v_tail() const { ++ if (!layers[0].v_cpu_per_stream.empty() && layers[0].v_cpu_per_stream[0]) { ++ return layers[0].v_cpu_per_stream[0]->type; ++ } ++ return type_v(); ++} ++ + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { + uint32_t result = 0; + +@@ -2800,6 +2829,26 @@ + return result; + } + ++// opencoti F5 M7 Stage 3c (#622 / #582-P2, bug-2161): dequant-on-lift a window/tail region view to f16 ++// for the POSITION_WINDOW concat-reassembly fallback. Mirrors dca.cpp:dca_lift_to_f16 exactly — the CUDA ++// CPY backend has direct ->f16 casts for f32/bf16/turbo but NO direct scalar-quant->f16, so scalar quants ++// route through an f32 intermediate (both legs GPU CPY kernels; d*q rounded to f16 is byte-identical to a ++// single-pass dequant). Needed only when the resident window and the spilled tail carry DIFFERENT KV types ++// (mixed-KV: q8 window ⊕ q4 tail) — ggml_concat asserts a->type==b->type, so lift both operands to f16 ++// before the concat. Uniform-type windows never call this (types match) → byte-identical to prior path. ++static ggml_tensor * poswin_lift_to_f16(ggml_context * ctx0, ggml_tensor * x) { ++ if (x->type == GGML_TYPE_F16) { ++ return x; ++ } ++ const bool scalar_quant = ++ x->type == GGML_TYPE_Q8_0 || x->type == GGML_TYPE_Q4_0 || x->type == GGML_TYPE_Q4_1 || ++ x->type == GGML_TYPE_Q5_0 || x->type == GGML_TYPE_Q5_1 || x->type == GGML_TYPE_Q6_0; ++ if (scalar_quant) { ++ x = ggml_cast(ctx0, x, GGML_TYPE_F32); ++ } ++ return ggml_cast(ctx0, x, GGML_TYPE_F16); ++} ++ + ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + +@@ -2884,6 +2933,13 @@ + ggml_row_size(kc_s->type, n_embd_k_gqa2), + ggml_row_size(kc_s->type, n_embd_k_gqa2 * (uint64_t) kc_s->ne[1]), + 0); ++ // #622 P2 mixed-KV: window and tail may carry different types (e.g. q8 window ⊕ q4 tail). ++ // ggml_concat asserts identical types, so lift both region views to f16 first. Uniform ++ // types (the common case, incl. the REG sentinel) skip this → byte-identical concat. ++ if (k_s->type != kc_s->type) { ++ win_v = poswin_lift_to_f16(ctx, win_v); ++ tail_v = poswin_lift_to_f16(ctx, tail_v); ++ } + return ggml_concat(ctx, win_v, tail_v, 2); + }; + +@@ -2997,6 +3053,11 @@ + ggml_row_size(vc_s->type, n_embd_v_gqa2), + ggml_row_size(vc_s->type, n_embd_v_gqa2 * (uint64_t) vc_s->ne[1]), + 0); ++ // #622 P2 mixed-KV: lift both region views to f16 when window/tail types differ (see get_k). ++ if (v_s->type != vc_s->type) { ++ win_v = poswin_lift_to_f16(ctx, win_v); ++ tail_v = poswin_lift_to_f16(ctx, tail_v); ++ } + return ggml_concat(ctx, win_v, tail_v, 2); + }; + +@@ -5278,7 +5339,13 @@ + // resident cache and vice-versa (cross-residency-mode compatible). + auto * k_cpu = layer.k_cpu_per_stream[cr.strm]; + const uint32_t wc = layer.window_cells; +- GGML_ASSERT((uint64_t) ggml_row_size(k_cpu->type, n_embd_k_gqa) == k_size_row); ++ // opencoti #622 P2 (bug-2162): the host tail may carry a DIFFERENT (more ++ // compressed) type than the resident window (q8 window ⊕ q4 tail), so its ++ // per-cell row is k_cpu_row, not k_size_row. Emit the window prefix at ++ // k_size_row and the tail suffix at k_cpu_row; the reader is symmetric. When ++ // tail type == window type (uniform window) k_cpu_row == k_size_row → the ++ // byte layout is identical to the prior single-row-size path. ++ const uint64_t k_cpu_row = ggml_row_size(k_cpu->type, n_embd_k_gqa); + for (const auto & range : cr.data) { + const uint32_t a = range.first, b = range.second; + const uint32_t w_hi = b < wc ? b : wc; // window cells [a, w_hi) +@@ -5289,10 +5356,12 @@ + wrote += (size_t) (w_hi - a) * k_size_row; + } + if (b > t_lo) { +- io.write_tensor(k_cpu, (size_t) (t_lo - wc) * k_size_row, (size_t) (b - t_lo) * k_size_row); +- wrote += (size_t) (b - t_lo) * k_size_row; ++ io.write_tensor(k_cpu, (size_t) (t_lo - wc) * k_cpu_row, (size_t) (b - t_lo) * k_cpu_row); ++ wrote += (size_t) (b - t_lo) * k_cpu_row; + } +- GGML_ASSERT(wrote == (size_t) (b - a) * k_size_row && "3c-6 K window/tail parity"); ++ const size_t expect = (size_t) (w_hi > a ? w_hi - a : 0) * k_size_row ++ + (size_t) (b > t_lo ? b - t_lo : 0) * k_cpu_row; ++ GGML_ASSERT(wrote == expect && "3c-6 K window/tail parity"); + } + } else { + // Read each range of cells of k_size length and write out +@@ -5351,7 +5420,8 @@ + // from v_cpu, in logical cell order → byte-identical on-disk layout. + auto * v_cpu = layer.v_cpu_per_stream[cr.strm]; + const uint32_t wc = layer.window_cells; +- GGML_ASSERT((uint64_t) ggml_row_size(v_cpu->type, n_embd_v_gqa) == v_size_row); ++ // opencoti #622 P2 (bug-2162): mixed-KV — tail carries its own row size. ++ const uint64_t v_cpu_row = ggml_row_size(v_cpu->type, n_embd_v_gqa); + for (const auto & range : cr.data) { + const uint32_t a = range.first, b = range.second; + const uint32_t w_hi = b < wc ? b : wc; +@@ -5362,10 +5432,12 @@ + wrote += (size_t) (w_hi - a) * v_size_row; + } + if (b > t_lo) { +- io.write_tensor(v_cpu, (size_t) (t_lo - wc) * v_size_row, (size_t) (b - t_lo) * v_size_row); +- wrote += (size_t) (b - t_lo) * v_size_row; ++ io.write_tensor(v_cpu, (size_t) (t_lo - wc) * v_cpu_row, (size_t) (b - t_lo) * v_cpu_row); ++ wrote += (size_t) (b - t_lo) * v_cpu_row; + } +- GGML_ASSERT(wrote == (size_t) (b - a) * v_size_row && "3c-6 V window/tail parity"); ++ const size_t expect = (size_t) (w_hi > a ? w_hi - a : 0) * v_size_row ++ + (size_t) (b > t_lo ? b - t_lo : 0) * v_cpu_row; ++ GGML_ASSERT(wrote == expect && "3c-6 V window/tail parity"); + } + } else { + // Read each range of cells of v_size length and write out +@@ -5627,13 +5699,17 @@ + // consumes one row from the stream in logical order. + auto * k_cpu = layer.k_cpu_per_stream[strm]; + const uint32_t wc = layer.window_cells; ++ // opencoti #622 P2 (bug-2162): symmetric to the writer — the host tail may ++ // carry a more-compressed type, so read k_cpu_row bytes into the tail (not ++ // k_size_row). Identical to the prior path when tail type == window type. ++ const uint64_t k_cpu_row = ggml_row_size(k_cpu->type, n_embd_k_gqa); + for (uint32_t i = 0; i < cell_count; ++i) { + const uint32_t p = sinfo.is_contiguous() + ? (sinfo.head() + i) : (uint32_t) sinfo.idxs[0][i]; + if (p < wc) { + io.read_tensor(k, (size_t) p * k_size_row, k_size_row); + } else { +- io.read_tensor(k_cpu, (size_t) (p - wc) * k_size_row, k_size_row); ++ io.read_tensor(k_cpu, (size_t) (p - wc) * k_cpu_row, k_cpu_row); + } + } + } else if (sinfo.is_contiguous()) { +@@ -5714,13 +5790,15 @@ + // io.read_tensor idiom — one row per call, sequential stream order. + auto * v_cpu = layer.v_cpu_per_stream[strm]; + const uint32_t wc = layer.window_cells; ++ // opencoti #622 P2 (bug-2162): mixed-KV — tail read at its own row size. ++ const uint64_t v_cpu_row = ggml_row_size(v_cpu->type, n_embd_v_gqa); + for (uint32_t i = 0; i < cell_count; ++i) { + const uint32_t p = sinfo.is_contiguous() + ? (sinfo.head() + i) : (uint32_t) sinfo.idxs[0][i]; + if (p < wc) { + io.read_tensor(v, (size_t) p * v_size_row, v_size_row); + } else { +- io.read_tensor(v_cpu, (size_t) (p - wc) * v_size_row, v_size_row); ++ io.read_tensor(v_cpu, (size_t) (p - wc) * v_cpu_row, v_cpu_row); + } + } + } else if (sinfo.is_contiguous()) { +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -157,7 +157,15 @@ + // opencoti #551 sparse-attn — Quest block-selector KV-block + // size B_SEL (docs/features/sparse_attn.md). 0 = sparse-attn + // OFF: no kmin/kmax side-cache allocated (byte-identical). +- uint32_t sparse_attn_block_size = 0); ++ uint32_t sparse_attn_block_size = 0, ++ // opencoti #582-P2 mixed-KV — DISTINCT ggml_type for the position-window ++ // TAIL (the older [window_cells, kv_size) region). GGML_TYPE_COUNT ++ // sentinel = "same as type_k/type_v" ⇒ byte-identical to a uniform cache. ++ // A more-compressed tail keeps recent tokens exact (f16 window) while the ++ // history is low-bit, all resident. Consulted only in position-window mode ++ // (the tail tensor exists); ignored for a vanilla / M2-head-split cache. ++ ggml_type type_k_tail = GGML_TYPE_COUNT, ++ ggml_type type_v_tail = GGML_TYPE_COUNT); + + ~llama_kv_cache() = default; + +@@ -248,6 +256,12 @@ + ggml_type type_k() const; + ggml_type type_v() const; + ++ // opencoti #582-P2 mixed-KV — the position-window TAIL's ggml_type (may differ ++ // from type_k/type_v). Derived from the tail tensor; equals type_k/type_v when ++ // there is no distinct tail (no window mode, or tail type == window type). ++ ggml_type type_k_tail() const; ++ ggml_type type_v_tail() const; ++ + // + // graph_build API + // +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -35,7 +35,9 @@ + const layer_reuse_cb & reuse, + llama_memory_t mem_other, + const layer_share_cb & share, +- uint32_t sparse_attn_block_size) : hparams(model.hparams), unified(unified) { ++ uint32_t sparse_attn_block_size, ++ ggml_type type_k_tail, ++ ggml_type type_v_tail) : hparams(model.hparams), unified(unified) { + + // opencoti bug-858 dual-context MTP — split the source iSWA cache into its base/swa halves + // so each inner kv_cache shares with the matching half. GUARD: both stay nullptr when +@@ -84,7 +86,9 @@ + compute_reserve_mib, window_measure_pass, + n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, + // opencoti bug-858 dual-context MTP — forward the base-half source + share selector. +- mem_other_base, share, sparse_attn_block_size); ++ mem_other_base, share, sparse_attn_block_size, ++ // opencoti #582-P2 — forward the spilled-tail KV type to both halves. ++ type_k_tail, type_v_tail); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + +@@ -95,7 +99,9 @@ + compute_reserve_mib, window_measure_pass, + n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, + // opencoti bug-858 dual-context MTP — forward the swa-half source + share selector. +- mem_other_swa, share, sparse_attn_block_size); ++ mem_other_swa, share, sparse_attn_block_size, ++ // opencoti #582-P2 — forward the spilled-tail KV type to both halves. ++ type_k_tail, type_v_tail); + } + + void llama_kv_cache_iswa::clear(bool data) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -62,7 +62,12 @@ + // opencoti #551 sparse-attn — forwarded to both kv_base and + // kv_swa. Quest block-selector B_SEL; 0 = off (no side-cache). + // See docs/features/sparse_attn.md. +- uint32_t sparse_attn_block_size = 0); ++ uint32_t sparse_attn_block_size = 0, ++ // opencoti #582-P2 (position-axis mixed KV) — distinct spilled-tail ++ // KV type, forwarded to both inner caches. GGML_TYPE_COUNT = same ++ // as window (byte-identical). See docs/evaluations/hal_kv_compression.md. ++ ggml_type type_k_tail = GGML_TYPE_COUNT, ++ ggml_type type_v_tail = GGML_TYPE_COUNT); + + ~llama_kv_cache_iswa() = default; + +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -2151,7 +2151,11 @@ + /* mem_other */ mtp_mem_other, + /* share */ mtp_share, + // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) +- cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); ++ cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0, ++ // opencoti #582-P2 — distinct spilled-tail KV type ++ // (GGML_TYPE_COUNT = same as window → byte-identical). ++ cparams.type_k_tail, ++ cparams.type_v_tail); + } else { + GGML_ASSERT(!hparams.is_swa_any()); + +@@ -2190,7 +2194,11 @@ + /* mem_other */ nullptr, + /* share */ nullptr, + // opencoti #551 sparse-attn — B_SEL when enabled, else 0 (off) +- cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0); ++ cparams.sparse_attn_enabled ? cparams.sparse_attn_block_size : 0, ++ // opencoti #582-P2 — distinct spilled-tail KV type ++ // (GGML_TYPE_COUNT = same as window → byte-identical). ++ cparams.type_k_tail, ++ cparams.type_v_tail); + } + } + } diff --git a/patches/0121-p2-auto-tail.patch b/patches/0121-p2-auto-tail.patch new file mode 100644 index 0000000000000000000000000000000000000000..df0c8dfe276a8177bf74199997ca64e73fed3d02 --- /dev/null +++ b/patches/0121-p2-auto-tail.patch @@ -0,0 +1,118 @@ +opencoti patch 0121 — P2-auto: auto-select a q4_0-floor compressed tail when the boot auto-tier is forced to spill (#653 / #582-P2) + +Extends the P1 boot auto-tier (patch 0116/0118, opencoti_auto_select_kv_tier) so +that when the auto-selected KV window still overflows the VRAM budget — i.e. the +rolling-KV path is FORCED to spill a tail regardless — the spilled tail [wc,n_kv) +can be given a MORE-compressed type than the resident window, cheaply reclaiming +H2D bytes on the oldest (least-attended) cells. It reuses the 0120 tail plumbing +(type_k_tail/type_v_tail, GGML_TYPE_COUNT sentinel = "tail == window"); this patch +only adds the boot POLICY that fills those sentinels in. + +Opt-in is SEPARATE from P1: OPENCOTI_KV_AUTO_TIER_TAIL=1 (in addition to +OPENCOTI_KV_AUTO_TIER=1). With P1 alone the tail stays == window, so a P1 boot is +byte-identical to before this patch — the compressed tail only engages when the +operator also asks for it. An explicit -ctkt/-ctvt (tail != COUNT) is honored and +never overridden, mirroring how -ctk/-ctv block the window auto-tier. + +Policy — "q4_0 floor when window > q4_0": for each axis independently, if the +chosen window type is higher-bit than q4_0 (f32/f16/bf16/q8_0/q6_0/q5_1/q5_0) and +that axis's tail is unset, set the tail to q4_0; otherwise leave tail == window +(so a window already at/below q4_0 is not "upgraded" to a coarser tail). The +policy hook (maybe_auto_tail) fires ONLY on the two spill exits — the "window +kept but spills" T* branch (chosen==0) and the "chosen tier still spills" +tail — so it never touches the fully-resident fast path (early return). + +Gate (bs2, Qwen2.5-14B-Instruct-1M-Q8_0, ctx 24576, vram-target 18000, -flash-attn +on): P2AUTO boot log shows `auto KV tail = q4_0/q4_0` over a real ~189 MiB / 3072-cell +spilled region (window q5_1/q4_0 second-pass binding budget); logit-equiv vs the +full-resident gold REF real_frac=0 (top-20 distribution equivalent, mean_tv 0.0087, +not worse than the uniform-tail P1AUTO floor). P1AUTO (OPENCOTI_KV_AUTO_TIER=1 only) +shows NO tail line — regression shield: byte-identical to the shipped P1 boot. + +Host-side only (src/llama-kv-cache.cpp is libllama in the host binary, not the CUDA +DSO): `make` rebuild, DSO ccache-hits. Marker: reuses the existing 0116 +`opencoti-hook: kv-auto-tier` site (this edit lives inside opencoti_auto_select_kv_tier +and its one call, exactly like 0118) — no new inline hook site. See +docs/features/rolling_kv.md and docs/protocols/UPSTREAM_SYNC.md (#653). + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index dc1273f..fdc3ad3 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -421,6 +421,7 @@ static float opencoti_host_ram_bw_gbps() { + static void opencoti_auto_select_kv_tier( + const llama_model & model, const llama_hparams & hparams, + ggml_type & type_k, ggml_type & type_v, ++ ggml_type & type_k_tail, ggml_type & type_v_tail, + bool v_trans, bool offload, uint32_t kv_size, uint32_t n_stream, + uint32_t vram_target_mib, llama_swa_type swa_type, float pcie_bw_gbps, + const std::function & filter) { +@@ -486,6 +487,35 @@ static void opencoti_auto_select_kv_tier( + return pc * (size_t) n_stream; + }; + ++ // opencoti #582 P2-auto (#622/#653) — when auto-tier is FORCED to spill (the chosen window ++ // still overflows the budget), optionally auto-pick a MORE-compressed type for the rolling-KV ++ // spilled TAIL than the resident window. SEPARATE opt-in (OPENCOTI_KV_AUTO_TIER_TAIL=1) so P1 ++ // boot stays byte-identical unless the operator also asks for a compressed tail. Policy: q4_0 ++ // floor — if a window axis is higher-bit than q4_0 AND its tail is still unset (GGML_TYPE_COUNT ++ // sentinel = "tail == window"), set that axis's tail to q4_0; else leave tail == window. Only ++ // reached with a real spill (this returns early on the fully-resident fast path), so the tail is ++ // always actually populated. Validated pair: q8-window ⊕ q4-tail (#622 S7). An explicit -ctkt/-ctvt ++ // (tail != COUNT) is honored as a block, mirroring -ctk/-ctv. ++ const bool tail_auto = getenv("OPENCOTI_KV_AUTO_TIER_TAIL") != nullptr; ++ auto higher_than_q4_0 = [](ggml_type w) -> bool { ++ return w == GGML_TYPE_F32 || w == GGML_TYPE_F16 || w == GGML_TYPE_BF16 || ++ w == GGML_TYPE_Q8_0 || w == GGML_TYPE_Q6_0 || w == GGML_TYPE_Q5_1 || w == GGML_TYPE_Q5_0; ++ }; ++ auto maybe_auto_tail = [&](ggml_type wk, ggml_type wv) { ++ if (!tail_auto) { ++ return; ++ } ++ if (type_k_tail == GGML_TYPE_COUNT && higher_than_q4_0(wk)) { type_k_tail = GGML_TYPE_Q4_0; } ++ if (type_v_tail == GGML_TYPE_COUNT && higher_than_q4_0(wv)) { type_v_tail = GGML_TYPE_Q4_0; } ++ if (type_k_tail != GGML_TYPE_COUNT || type_v_tail != GGML_TYPE_COUNT) { ++ LLAMA_LOG_WARN("%s: auto KV tail = %s/%s (compressed spilled tail; window %s/%s) — " ++ "set -ctkt/-ctvt to override\n", __func__, ++ type_k_tail == GGML_TYPE_COUNT ? ggml_type_name(wk) : ggml_type_name(type_k_tail), ++ type_v_tail == GGML_TYPE_COUNT ? ggml_type_name(wv) : ggml_type_name(type_v_tail), ++ ggml_type_name(wk), ggml_type_name(wv)); ++ } ++ }; ++ + // Ladder: least → most compression (HAL §5 P1 / §2 quality bar). q8_0/q4_0 is the + // recommended hi-K/cheap-V recipe (§2); q5_1/q4_0 is the last-resort floor. + struct kv_tier { ggml_type k, v; }; +@@ -602,17 +632,22 @@ static void opencoti_auto_select_kv_tier( + "of %.0f MiB free) — f16-window preferred over quantizing; set -ctk/-ctv to override\n", + __func__, ggml_type_name(ladder[0].k), ggml_type_name(ladder[0].v), + (kvb0 - budget) / 1048576.0, budget / 1048576.0, free_vram / 1048576.0); ++ maybe_auto_tail(ladder[0].k, ladder[0].v); // window kept but spills → compress the tail (P2-auto) + } + return; + } + type_k = ladder[chosen].k; + type_v = ladder[chosen].v; + const size_t kv_bytes = per_cell_bytes(type_k, type_v) * (size_t) kv_size; ++ const bool window_spills = kv_bytes > budget; + LLAMA_LOG_WARN("%s: auto KV tier = %s/%s (KV %.0f MiB %s budget %.0f MiB of %.0f MiB free); " + "set -ctk/-ctv to override\n", __func__, + ggml_type_name(type_k), ggml_type_name(type_v), +- kv_bytes / 1048576.0, kv_bytes <= budget ? "fits" : "OVER (window spills)", ++ kv_bytes / 1048576.0, window_spills ? "OVER (window spills)" : "fits", + budget / 1048576.0, free_vram / 1048576.0); ++ if (window_spills) { ++ maybe_auto_tail(type_k, type_v); // chosen tier still spills → compress the tail (P2-auto) ++ } + } + + llama_kv_cache::llama_kv_cache( +@@ -708,7 +743,7 @@ llama_kv_cache::llama_kv_cache( + // consumes type_k/type_v (the auto-frac + window sizers below, and the per-layer tensor alloc). + // Opt-in (OPENCOTI_KV_AUTO_TIER=1) + user-default -ctk/-ctv + dense full-attention only; leaves + // type_k/type_v untouched (byte-identical) in every other case. See opencoti_auto_select_kv_tier. +- opencoti_auto_select_kv_tier(model, hparams, type_k, type_v, v_trans, offload, ++ opencoti_auto_select_kv_tier(model, hparams, type_k, type_v, type_k_tail, type_v_tail, v_trans, offload, + kv_size, n_stream, vram_target_mib, swa_type, pcie_bw_gbps, filter); + + // opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md diff --git a/patches/0122-rys-layer-duplication.patch b/patches/0122-rys-layer-duplication.patch new file mode 100644 index 0000000000000000000000000000000000000000..c2be243433f62707eb1975a301fa62f9c2831287 --- /dev/null +++ b/patches/0122-rys-layer-duplication.patch @@ -0,0 +1,1779 @@ +opencoti patch 0122 — RYS layer duplication (--repeat-layers): weight-shared layer re-run across all text archs (Qwen dense/MoE, Gemma-4 iSWA dense+A4B, recurrent) + boundary-layer advisory (bug-2164) + +Adds RYS (Repeat-Your-Self) layer duplication: a boot-time --repeat-layers plan +that re-runs selected transformer layers with SHARED weights, growing the +effective depth (n_layer_eff) of a model without adding parameters. The plan is +parsed host-side into cparams, validated, and driven through an effective→source +layer map so the per-layer forward loop, the KV cache allocation, and the +iSWA/recurrent/hybrid memory wrappers all address layers by effective index while +reusing the source layer's weights and KV mapping (map_layer_ids extension). It +spans every text arch family: Qwen dense (qwen2/qwen3), Qwen MoE +(qwen2moe/qwen3moe/qwen35moe), Qwen3-Next recurrent-hybrid, and Gemma-4 dense +(12B/31B) + A4B under iSWA dual-cache. A boundary-layer advisory (first/last +max(3,n/8) layers) warns at boot that duplicating boundary layers of +franken-merge/passthrough models can degrade coherence (bug-2164 — not-a-bug: +model fragility, not an SWA engine bug), while middle-layer duplication ships +clean. All additive; guarded so a model without --repeat-layers is byte-identical. + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +index ae386ce6c..958e0db41 100644 +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1539,6 +1539,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.dca_yarn_factor = std::stof(value); + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_DCA_YARN_FACTOR")); ++ // opencoti RYS (#659) — runtime weight-shared layer duplication — see docs/features/rys_layer_duplication.md ++ add_opt(common_arg( ++ {"--repeat-layers"}, "SPEC", ++ "RYS runtime layer duplication (weight-shared, no re-quant, no merged GGUF). SPEC = one or more " ++ "`;`-separated blocks `i,j` (or `i-j`), each re-running SOURCE transformer layers [i,j) a second time " ++ "IN PLACE with the same weight tensors (zero extra param VRAM; pay only KV + compute of the extra " ++ "passes). Example: --repeat-layers 26,34 (Qwen3.5-27B RYS-XL, +8 layers) or 33,34 (RYS-S, +1). " ++ "MVP: dense full-attention (Qwen3/3.5/3.6 dense) only; hard-errors on iSWA/hybrid/DCA/sparse. " ++ "Empty = off (byte-identical).", ++ [](common_params & params, const std::string & value) { ++ params.repeat_layers = value; // parsed + validated at load (llama-context ctor) ++ } ++ ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REPEAT_LAYERS")); + // opencoti #551 sparse-attn — Quest-style block-selector sparse attention — see docs/features/sparse_attn.md + add_opt(common_arg( + {"--sparse-attn"}, "MODE", +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +index cd9709883..f149e2931 100644 +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -1657,6 +1657,9 @@ struct llama_context_params common_context_params_to_llama(const common_params & + cparams.sparse_attn_sink = params.sparse_attn_sink; + cparams.sparse_attn_refresh = params.sparse_attn_refresh; // opencoti #551 PERF (was omitted — flag was inert) + cparams.sparse_attn_mode = params.sparse_attn_mode; // opencoti-hook: sparse-attn-vslash — #551 ++ // opencoti RYS (#659) — layer duplication spec. The C-ABI carries a const char*; it points into ++ // `params.repeat_layers` (alive through model+context construction). Empty ⇒ nullptr ⇒ off. ++ cparams.repeat_layers = params.repeat_layers.empty() ? nullptr : params.repeat_layers.c_str(); + cparams.n_rs_seq = params.speculative.need_n_rs_seq(); + cparams.n_batch = params.n_batch; + cparams.n_ubatch = params.n_ubatch; +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index 738962de3..d44d51d48 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -485,6 +485,11 @@ struct common_params { + // opencoti-hook: sparse-attn-vslash — #551 selector mechanism: 0 = Quest geometric kmin/kmax bound + // (default, byte-identical to prior behavior); 1 = vertical-slash content-driven max-logit selector. + uint32_t sparse_attn_mode = 0; ++ // opencoti RYS (#659) — runtime weight-shared layer duplication. Spec string of `;`-separated ++ // half-open blocks `i,j` (or `i-j`), each re-running source transformer layers [i,j) in place ++ // with SHARED weights (no extra param VRAM). "" = off (byte-identical). Dense Qwen3 only in the ++ // MVP; hard-errors at load on iSWA/hybrid/DCA/sparse. See docs/features/rys_layer_duplication.md. ++ std::string repeat_layers; + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index 6e4829d2c..20141023c 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -433,6 +433,14 @@ extern "C" { + // opencoti bug-858: dual-context MTP — target context whose KV the draft context shares (nullptr if none) + struct llama_context * ctx_other; + ++ // opencoti RYS (#656) — runtime weight-shared layer duplication. See ++ // docs/features/rys_layer_duplication.md. Spec string of `;`-separated ++ // half-open blocks `i,j` (or `i-j`), each duplicating source transformer ++ // layers [i,j) in place (weights shared by pointer). nullptr/"" = off ++ // (identity plan, byte-identical). Expanded to cparams.layer_plan and ++ // validated (0<=imctx->get_base(); + +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 83e30f485..72ff10248 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -14,6 +14,7 @@ + #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" + ++#include + #include + #include + #include +@@ -38,6 +39,82 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { + throw std::runtime_error("Unsupported ctx type"); + } + ++// opencoti RYS (#656): parse+expand+validate the --repeat-layers spec string into a ++// layer_plan (vector of SOURCE transformer-layer indices, length n_layer_eff). Weights ++// and per-layer arch props index by src; the KV slot handed to build_attn indexes by the ++// eff position. See docs/features/rys_layer_duplication.md. Logic proven byte-for-byte ++// against scratchpad/rys_plan.py + the 6 article Qwen3.5-27B Pareto configs. ++// ++// spec grammar: `;`-separated blocks, each `i,j` or `i-j` = half-open [i,j) source layers ++// duplicated in place after their last member. Validates 0 <= i < j <= n_transformer and ++// disjoint-ascending. Returns false with a human message in `err` on any malformed spec. ++// EMPTY spec (nullptr / "" / only separators) => out_plan stays EMPTY == OFF == identity ++// == byte-identical (the caller must treat an empty plan as "no duplication"). ++static bool opencoti_parse_repeat_layers(const char * spec, uint32_t n_transformer, ++ std::vector & out_plan, std::string & err) { ++ out_plan.clear(); ++ err.clear(); ++ if (spec == nullptr || *spec == '\0') { ++ return true; ++ } ++ std::string s(spec); ++ s.erase(std::remove(s.begin(), s.end(), ' '), s.end()); ++ std::vector> blocks; ++ size_t pos = 0; ++ while (pos <= s.size()) { ++ size_t semi = s.find(';', pos); ++ std::string part = (semi == std::string::npos) ? s.substr(pos) : s.substr(pos, semi - pos); ++ if (!part.empty()) { ++ size_t sep = part.find(','); ++ if (sep == std::string::npos) sep = part.find('-'); ++ if (sep == std::string::npos) { ++ err = "--repeat-layers block '" + part + "' needs 'i,j' or 'i-j'"; ++ return false; ++ } ++ int i, j; ++ try { ++ size_t ci = 0, cj = 0; ++ i = std::stoi(part.substr(0, sep), &ci); ++ j = std::stoi(part.substr(sep + 1), &cj); ++ if (ci != sep || cj != part.size() - sep - 1) throw std::invalid_argument("trailing"); ++ } catch (...) { ++ err = "--repeat-layers block '" + part + "' has non-integer bounds"; ++ return false; ++ } ++ blocks.emplace_back(i, j); ++ } ++ if (semi == std::string::npos) break; ++ pos = semi + 1; ++ } ++ for (const auto & b : blocks) { ++ if (!(b.first >= 0 && b.first < b.second && (uint32_t) b.second <= n_transformer)) { ++ err = "--repeat-layers block (" + std::to_string(b.first) + "," + std::to_string(b.second) + ++ ") out of range: need 0 <= i < j <= " + std::to_string(n_transformer) + ++ " (transformer layers, MTP head excluded)"; ++ return false; ++ } ++ } ++ std::sort(blocks.begin(), blocks.end()); ++ for (size_t k = 1; k < blocks.size(); ++k) { ++ if (blocks[k - 1].second > blocks[k].first) { ++ err = "--repeat-layers blocks overlap (MVP requires disjoint blocks)"; ++ return false; ++ } ++ } ++ if (blocks.empty()) { ++ return true; // spec was only separators -> treat as off ++ } ++ for (uint32_t src = 0; src < n_transformer; ++src) { ++ out_plan.push_back((int) src); ++ for (const auto & b : blocks) { ++ if ((int) src == b.second - 1) { ++ for (int d = b.first; d < b.second; ++d) out_plan.push_back(d); ++ } ++ } ++ } ++ return true; ++} ++ + llama_context::llama_context( + const llama_model & model, + llama_context_params params) : +@@ -160,6 +237,83 @@ llama_context::llama_context( + cparams.ctx_type = params.ctx_type; + cparams.ctx_other = params.ctx_other; // opencoti bug-858: dual-context MTP KV sharing + ++ // opencoti RYS (#656): expand the --repeat-layers spec into cparams.layer_plan. The plan ++ // domain is the TRANSFORMER layers only (n_layer minus the nextn/MTP head, which is driven ++ // by its own path and must never be duplicated). Empty spec => empty plan => identity => ++ // byte-identical. Any malformed / out-of-range / overlapping spec fails fast at construction ++ // (fail-loud is correct: a silently-ignored duplication request would mislead the operator). ++ // See docs/features/rys_layer_duplication.md. ++ { ++ const uint32_t n_transformer = hparams.n_layer - hparams.nextn_predict_layers; ++ std::string rys_err; ++ if (!opencoti_parse_repeat_layers(params.repeat_layers, n_transformer, cparams.layer_plan, rys_err)) { ++ throw std::runtime_error("opencoti RYS: " + rys_err); ++ } ++ if (!cparams.layer_plan.empty()) { ++ // opencoti RYS — production compatibility. Wired across ALL text archs: standard-attention ++ // DENSE + MoE (Qwen2/2.5/3/3-MoE), the iSWA dual-cache (Gemma-4 12B/31B dense + A4B; #665, ++ // eff-length base/swa + eff-aware reuse), AND the recurrent/hybrid state cache (Qwen3.5/3.6 ++ // gated-delta-net; #666, eff-length r_l/s_l + hybrid-wrapper layer_plan threading). ++ // ++ // #667 lifted the DCA / sparse-attn / rolling-KV-residency guards: the three residency ++ // SIZING loops (opencoti_compute_auto_gpu_heads_frac / _resident_window_cells / ++ // _auto_select_kv_tier) now iterate the EFFECTIVE plan (eff→src) so KV demand counts every ++ // duplicated slot; the DCA build path maps eff→src for the one per-source-layer hparam it ++ // reads (is_swa); sparse-attn alloc already lives inside the eff-iteration loop and reads ++ // only the global sparse_attn_block_size. Every per-layer window/tail/tactic tensor is ++ // keyed by `layers[]` + map_layer_ids (eff), so the storage + forward paths were already ++ // eff-correct. RYS off ⇒ empty plan ⇒ eff==src everywhere ⇒ byte-identical. ++ // See docs/features/rys_layer_duplication.md §6. ++ ++ // Boot-log plan dump — announce audibly + emit the full eff->src map so the operator can ++ // verify the duplication seam matches the requested blocks (validation gate §9.2). ++ LLAMA_LOG_WARN("%s: RYS layer duplication ENGAGED: --repeat-layers '%s' -> %u transformer " ++ "layers -> %zu effective layers (+%zu; weights shared by pointer, extra KV + " ++ "compute only). See docs/features/rys_layer_duplication.md\n", ++ __func__, params.repeat_layers, n_transformer, cparams.layer_plan.size(), ++ cparams.layer_plan.size() - n_transformer); ++ std::string plan_dump; ++ plan_dump.reserve(cparams.layer_plan.size() * 4); ++ for (size_t eff = 0; eff < cparams.layer_plan.size(); ++eff) { ++ if (eff) plan_dump += ','; ++ plan_dump += std::to_string(cparams.layer_plan[eff]); ++ } ++ LLAMA_LOG_INFO("%s: RYS eff->src plan: [%s]\n", __func__, plan_dump.c_str()); ++ ++ // opencoti RYS (bug-2161): boundary-layer advisory. Duplicating the FIRST or LAST ++ // transformer layers reliably produces incoherent output — this is a MODEL property ++ // (the same fragility franken-merge / passthrough self-merge tools avoid by never ++ // duplicating boundary layers: mergekit passthrough, SOLAR depth-upscaling), NOT an ++ // engine defect. Verified 2026-07-13 on native Gemma-4-26B-A4B: the coherent band was ++ // L4..~L24, while L0-L3 (early) AND L25-L29 (late) all decoded garbage — independent of ++ // SWA/global (middle SWA layers L4/L10 are coherent; boundary GLOBAL L29 is garbage). ++ // Qwen3-8B has a wider tolerant band (only its last layer broke). We warn but do NOT ++ // block: the exact usable band is model-dependent and the operator may know better. ++ { ++ std::vector dup_count(n_transformer, 0); ++ for (uint32_t src : cparams.layer_plan) { ++ if (src < n_transformer) dup_count[src]++; ++ } ++ const uint32_t band = std::max(3, n_transformer / 8); ++ std::string boundary_dups; ++ for (uint32_t src = 0; src < n_transformer; ++src) { ++ if (dup_count[src] > 1 && (src < band || src >= n_transformer - band)) { ++ if (!boundary_dups.empty()) boundary_dups += ','; ++ boundary_dups += std::to_string(src); ++ } ++ } ++ if (!boundary_dups.empty()) { ++ LLAMA_LOG_WARN("%s: RYS ADVISORY — plan duplicates BOUNDARY layer(s) [%s] " ++ "(within the first/last %u of %u). Duplicating early/late layers " ++ "commonly yields incoherent output (a model property, not a bug); " ++ "prefer MIDDLE layers [%u..%u]. See rys_layer_duplication.md.\n", ++ __func__, boundary_dups.c_str(), band, n_transformer, ++ band, n_transformer - band - 1); ++ } ++ } ++ } ++ } ++ + // Initialize backend samplers here so they are part of the sampling graph + // before the reserve passes run later in this function. This avoids a later + // re-reserve when graph nodes change. +@@ -661,7 +815,12 @@ void llama_context::sched_reserve() { + if (strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) != 0) { + continue; + } +- const int il = std::stoi(n->name + prefix_len); ++ // opencoti RYS (#666): FA / fused-GDN ops are named with the EFFECTIVE (KV/state ++ // slot) layer index, but model.dev_layer() is keyed by SOURCE layer. Under ++ // --repeat-layers a duplicated layer has eff >= n_layer, so dev_layer(eff) throws ++ // std::out_of_range. Map eff→src (the device is the source layer's device). ++ const int il_name = std::stoi(n->name + prefix_len); ++ const int il = cparams.layer_plan.empty() ? il_name : cparams.layer_plan[il_name]; + ggml_backend_dev_t device_kv = model.dev_layer(il); + if (device_fa != device_kv) { + LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor " +@@ -703,7 +862,8 @@ void llama_context::sched_reserve() { + ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); + + GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0); +- const int il = std::stoi(n->name + prefix_len); ++ const int il_name = std::stoi(n->name + prefix_len); ++ const int il = cparams.layer_plan.empty() ? il_name : cparams.layer_plan[il_name]; // opencoti RYS (#666): eff→src for dev_layer + ggml_backend_dev_t device_kv = model.dev_layer(il); + if (device_gdn != device_kv) { + LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " +@@ -744,7 +904,8 @@ void llama_context::sched_reserve() { + ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); + + GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0); +- const int il = std::stoi(n->name + prefix_len); ++ const int il_name = std::stoi(n->name + prefix_len); ++ const int il = cparams.layer_plan.empty() ? il_name : cparams.layer_plan[il_name]; // opencoti RYS (#666): eff→src for dev_layer + ggml_backend_dev_t device_kv = model.dev_layer(il); + if (device_gdn != device_kv) { + LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " +@@ -3784,6 +3945,7 @@ llama_context_params llama_context_default_params() { + /*.abort_callback =*/ nullptr, + /*.abort_callback_data =*/ nullptr, + /*.ctx_other =*/ nullptr, // opencoti bug-858: dual-context MTP ++ /*.repeat_layers =*/ nullptr, // opencoti RYS (#656): layer duplication off by default + /*.embeddings =*/ false, + /*.offload_kqv =*/ true, + /*.no_perf =*/ true, +diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h +index 030334820..4308471e3 100644 +--- a/llama.cpp/src/llama-cparams.h ++++ b/llama.cpp/src/llama-cparams.h +@@ -3,6 +3,7 @@ + #include "llama.h" + + #include ++#include + + #define LLAMA_MAX_SEQ 256 + +@@ -125,4 +126,11 @@ struct llama_cparams { + // draft (MTP) context shares IN-PLACE (upstream cparams.ctx_other + mem_other). + // nullptr for ordinary contexts. + llama_context * ctx_other; ++ ++ // opencoti RYS (#656): runtime weight-shared layer duplication — see ++ // docs/features/rys_layer_duplication.md. layer_plan[eff] = SOURCE layer ++ // index; weights/arch props index by src, KV-slot by eff. EMPTY == OFF == ++ // identity == byte-identical (the ctor only populates it when the ++ // llama_context_params.repeat_layers spec string is non-empty and valid). ++ std::vector layer_plan; + }; +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index 874fa9017..4593339c9 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -2767,7 +2767,12 @@ ggml_tensor * llm_graph_context::build_attn( + ggml_build_forward_expand(gf, k_cur); + ggml_build_forward_expand(gf, v_cur); + +- const bool is_swa = hparams.is_swa(il); ++ // opencoti RYS (#667): map eff→src for the per-SOURCE-layer swa-ness (is_swa GGML_ABORTs for ++ // il >= n_layer, which a duplicated eff can hit). Mirrors the cached build_attn_mha fix. Empty ++ // plan ⇒ il_src == il ⇒ byte-identical. (No-cache path — rarely combined with RYS, hardened for ++ // completeness so every attention builder is uniformly eff-safe.) ++ const int il_src = cparams.layer_plan.empty() ? il : cparams.layer_plan[il]; ++ const bool is_swa = hparams.is_swa(il_src); + + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + +@@ -3202,7 +3207,14 @@ ggml_tensor * llm_graph_context::build_attn( + ggml_tensor * v_mla, + float kq_scale, + int il) const { +- const bool is_swa = hparams.is_swa(il); ++ // opencoti RYS (#665): `il` is the EFFECTIVE KV-slot (the graph passes eff; map_layer_ids + ++ // cpy_k/cpy_v are eff-keyed, so `il` MUST stay eff below). But the base-vs-SWA inner-cache ++ // selection — and the k/v idx + rot choice — is a MODEL property keyed by the SOURCE layer: ++ // a duplicated layer keeps its source's swa-ness, and eff can even exceed n_layer (is_swa(eff) ++ // would GGML_ABORT). Resolve src via the plan for is_swa only. RYS off ⇒ plan empty ⇒ src==il ++ // ⇒ byte-identical. ++ const int il_src = cparams.layer_plan.empty() ? il : cparams.layer_plan[il]; ++ const bool is_swa = hparams.is_swa(il_src); + + auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot; + auto * v_rot = is_swa ? inp->self_v_rot_swa : inp->self_v_rot; +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index bdf457e0b..584370965 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -37,7 +37,8 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + const layer_share_cb & share, + uint32_t sparse_attn_block_size, + ggml_type type_k_tail, +- ggml_type type_v_tail) : hparams(model.hparams), unified(unified) { ++ ggml_type type_v_tail, ++ const std::vector & layer_plan) : hparams(model.hparams), unified(unified) { + + // opencoti bug-858 dual-context MTP — split the source iSWA cache into its base/swa halves + // so each inner kv_cache shares with the matching half. GUARD: both stay nullptr when +@@ -88,7 +89,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + // opencoti bug-858 dual-context MTP — forward the base-half source + share selector. + mem_other_base, share, sparse_attn_block_size, + // opencoti #582-P2 — forward the spilled-tail KV type to both halves. +- type_k_tail, type_v_tail); ++ type_k_tail, type_v_tail, ++ // opencoti RYS (#665) — forward the eff→src duplication plan to the base half. ++ layer_plan); + + LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); + +@@ -101,7 +104,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( + // opencoti bug-858 dual-context MTP — forward the swa-half source + share selector. + mem_other_swa, share, sparse_attn_block_size, + // opencoti #582-P2 — forward the spilled-tail KV type to both halves. +- type_k_tail, type_v_tail); ++ type_k_tail, type_v_tail, ++ // opencoti RYS (#665) — forward the eff→src duplication plan to the swa half. ++ layer_plan); + } + + void llama_kv_cache_iswa::clear(bool data) { +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 014af5bc6..5018928f4 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -67,7 +67,13 @@ public: + // KV type, forwarded to both inner caches. GGML_TYPE_COUNT = same + // as window (byte-identical). See docs/evaluations/hal_kv_compression.md. + ggml_type type_k_tail = GGML_TYPE_COUNT, +- ggml_type type_v_tail = GGML_TYPE_COUNT); ++ ggml_type type_v_tail = GGML_TYPE_COUNT, ++ // opencoti RYS (#665) — runtime weight-shared layer duplication plan ++ // (eff→src). Forwarded verbatim to both inner kv_base/kv_swa so each ++ // half allocates n_layer_eff slots keyed by eff, partitioned by the ++ // SOURCE layer's is_swa. Empty = identity = byte-identical (default). ++ // See docs/features/rys_layer_duplication.md. ++ const std::vector & layer_plan = {}); + + ~llama_kv_cache_iswa() = default; + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index fdc3ad331..932500436 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -151,6 +151,7 @@ static float opencoti_compute_auto_gpu_heads_frac( + uint32_t n_stream, + uint32_t vram_target_mib, + size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back ++ const std::vector & layer_plan, // opencoti RYS (#667): eff→src plan; empty = identity + const std::function & filter) { + // No GPU offload → nothing is "GPU-resident" and nothing to spill; the M2 + // split is only meaningful for offloaded layers. Stay at vanilla. +@@ -162,7 +163,12 @@ static float opencoti_compute_auto_gpu_heads_frac( + ggml_backend_dev_t dev = nullptr; + ggml_backend_dev_t dev_prev = nullptr; + int n_dev_distinct = 0; +- for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ // opencoti RYS (#667): iterate the EFFECTIVE layer plan so the KV demand counts every ++ // duplicated slot (each duplicate's bytes == its SOURCE layer's), mirroring the ctor's ++ // n_layer_eff alloc loop. Empty plan ⇒ eff==il==src ⇒ byte-identical to the pre-RYS sum. ++ const uint32_t rys_n_eff = layer_plan.empty() ? hparams.n_layer : (uint32_t) layer_plan.size(); ++ for (uint32_t rys_eff = 0; rys_eff < rys_n_eff; ++rys_eff) { ++ const uint32_t il = layer_plan.empty() ? rys_eff : (uint32_t) layer_plan[rys_eff]; // SOURCE layer + if (!hparams.has_kv(il)) { + continue; + } +@@ -259,6 +265,7 @@ static uint32_t opencoti_compute_resident_window_cells( + uint32_t n_pad, + size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back + bool measure_pass, // opencoti bug-1342: pass 1 → minimal window (tiny device KV) ++ const std::vector & layer_plan, // opencoti RYS (#667): eff→src plan; empty = identity + const std::function & filter) { + if (!offload || kv_size == 0) { + return kv_size; // no position-axis spill without GPU offload +@@ -275,7 +282,11 @@ static uint32_t opencoti_compute_resident_window_cells( + + size_t per_cell = 0; // bytes for ONE position cell across all this cache's KV layers + ggml_backend_dev_t dev = nullptr; +- for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ // opencoti RYS (#667): eff→src iteration — per_cell must sum EVERY effective KV slot (each ++ // duplicate costs its source layer's bytes). Empty plan ⇒ byte-identical to the pre-RYS sum. ++ const uint32_t rys_n_eff = layer_plan.empty() ? hparams.n_layer : (uint32_t) layer_plan.size(); ++ for (uint32_t rys_eff = 0; rys_eff < rys_n_eff; ++rys_eff) { ++ const uint32_t il = layer_plan.empty() ? rys_eff : (uint32_t) layer_plan[rys_eff]; // SOURCE layer + if (!hparams.has_kv(il)) { + continue; + } +@@ -424,6 +435,7 @@ static void opencoti_auto_select_kv_tier( + ggml_type & type_k_tail, ggml_type & type_v_tail, + bool v_trans, bool offload, uint32_t kv_size, uint32_t n_stream, + uint32_t vram_target_mib, llama_swa_type swa_type, float pcie_bw_gbps, ++ const std::vector & layer_plan, // opencoti RYS (#667): eff→src plan; empty = identity + const std::function & filter) { + if (getenv("OPENCOTI_KV_AUTO_TIER") == nullptr) { + return; // opt-in only — silent + byte-identical when unset +@@ -448,7 +460,11 @@ static void opencoti_auto_select_kv_tier( + // Budget — identical math to opencoti_compute_resident_window_cells (type-independent). + ggml_backend_dev_t dev = nullptr; + int32_t il0 = -1; // opencoti #582 T4 — first KV layer, for the FA compute microbench +- for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ // opencoti RYS (#667): first KV layer by EFFECTIVE order → its SOURCE index (valid model layer ++ // for the microbench + device). Empty plan ⇒ eff==src ⇒ identical to the pre-RYS first-KV scan. ++ const uint32_t rys_n_eff_a = layer_plan.empty() ? hparams.n_layer : (uint32_t) layer_plan.size(); ++ for (uint32_t rys_eff = 0; rys_eff < rys_n_eff_a; ++rys_eff) { ++ const uint32_t il = layer_plan.empty() ? rys_eff : (uint32_t) layer_plan[rys_eff]; // SOURCE layer + if (!hparams.has_kv(il) || (filter && !filter(il))) { + continue; + } +@@ -475,7 +491,10 @@ static void opencoti_auto_select_kv_tier( + // Per-cell KV bytes for a candidate (tk,tv) — same per-layer sum as the residency sizer. + auto per_cell_bytes = [&](ggml_type tk, ggml_type tv) -> size_t { + size_t pc = 0; +- for (uint32_t il = 0; il < hparams.n_layer; ++il) { ++ // opencoti RYS (#667): eff→src — sum every effective KV slot at candidate (tk,tv). ++ const uint32_t rys_n_eff_b = layer_plan.empty() ? hparams.n_layer : (uint32_t) layer_plan.size(); ++ for (uint32_t rys_eff = 0; rys_eff < rys_n_eff_b; ++rys_eff) { ++ const uint32_t il = layer_plan.empty() ? rys_eff : (uint32_t) layer_plan[rys_eff]; // SOURCE layer + if (!hparams.has_kv(il) || (filter && !filter(il))) { + continue; + } +@@ -676,7 +695,8 @@ llama_kv_cache::llama_kv_cache( + const layer_share_cb & share, + uint32_t sparse_attn_block_size, + ggml_type type_k_tail, +- ggml_type type_v_tail) : ++ ggml_type type_v_tail, ++ const std::vector & layer_plan) : // opencoti RYS (#658) — eff→src plan; empty = identity + model(model), hparams(model.hparams), v_trans(v_trans), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type), + // opencoti bug-858 dual-context MTP — `other` is the TARGET cache (nullptr for every +@@ -739,12 +759,22 @@ llama_kv_cache::llama_kv_cache( + + const uint32_t n_layer_kv = hparams.n_layer_kv(); + ++ // opencoti RYS (#658) — runtime weight-shared layer duplication. When layer_plan is non-empty the ++ // cache allocates one distinct slot per EFFECTIVE position (plan.size()), each sized from its ++ // SOURCE layer's dims. n_layer_eff drives the per-(buft,stream) metadata-ctx budget below and the ++ // alloc loop's iteration domain. EMPTY plan (every non-dense-Qwen caller / RYS off) ⇒ n_layer_eff ++ // falls back so all sizing + naming is byte-identical. See docs/features/rys_layer_duplication.md. ++ const uint32_t n_layer_eff = layer_plan.empty() ? hparams.n_layer : (uint32_t) layer_plan.size(); ++ // Metadata-ctx budget must cover the effective (post-duplication) layer count. n_layer_eff >= ++ // n_layer_kv whenever RYS is on (duplication only adds); off-path this is exactly n_layer_kv. ++ const uint32_t n_layer_ctx_budget = std::max(n_layer_kv, n_layer_eff); ++ + // opencoti HAL #582 P1 (#621) — auto KV-tier selection, run BEFORE any residency/allocation math + // consumes type_k/type_v (the auto-frac + window sizers below, and the per-layer tensor alloc). + // Opt-in (OPENCOTI_KV_AUTO_TIER=1) + user-default -ctk/-ctv + dense full-attention only; leaves + // type_k/type_v untouched (byte-identical) in every other case. See opencoti_auto_select_kv_tier. + opencoti_auto_select_kv_tier(model, hparams, type_k, type_v, type_k_tail, type_v_tail, v_trans, offload, +- kv_size, n_stream, vram_target_mib, swa_type, pcie_bw_gbps, filter); ++ kv_size, n_stream, vram_target_mib, swa_type, pcie_bw_gbps, layer_plan, filter); + + // opencoti F5 M7 Rolling KV — Rung 0 auto-residency — see docs/features/rolling_kv.md + // Resolve the effective HeadInfer fraction once for this cache. A negative +@@ -761,7 +791,7 @@ llama_kv_cache::llama_kv_cache( + const float eff_gpu_heads_frac = headinfer_gpu_heads_frac < 0.0f + ? opencoti_compute_auto_gpu_heads_frac(model, hparams, type_k, type_v, + v_trans, offload, kv_size, n_stream, vram_target_mib, +- opencoti_compute_reserve_bytes, filter) ++ opencoti_compute_reserve_bytes, layer_plan, filter) + : headinfer_gpu_heads_frac; + + // opencoti F5 M7 Stage 3c — position-axis window sizing. Computed + logged now so the +@@ -771,7 +801,7 @@ llama_kv_cache::llama_kv_cache( + // value is intentionally unused downstream at this sub-stage. See docs/features/advanced_kv.md. + const uint32_t window_cells = opencoti_compute_resident_window_cells( + model, hparams, type_k, type_v, v_trans, offload, kv_size, n_stream, +- vram_target_mib, n_pad, opencoti_compute_reserve_bytes, window_measure_pass, filter); ++ vram_target_mib, n_pad, opencoti_compute_reserve_bytes, window_measure_pass, layer_plan, filter); + // opencoti F5 M7 Stage 4 (#338) — position-window mode is driven by the + // --kv-residency-mode knob (cparams.kv_residency_mode): 0=auto, 1=head, 2=window. + // The ELIGIBILITY predicate is the validated POSITION_WINDOW class: the sizer found a +@@ -876,7 +906,7 @@ llama_kv_cache::llama_kv_cache( + // with margin). no_alloc ⇒ metadata only, so the KV data buffer is + // unaffected; off (block_size==0) keeps the original 4u (byte-identical). + ggml_init_params params = { +- /*.mem_size =*/ size_t((sparse_attn_block_size > 0 ? 8u : 4u)*n_layer_kv*ggml_tensor_overhead()), ++ /*.mem_size =*/ size_t((sparse_attn_block_size > 0 ? 8u : 4u)*n_layer_ctx_budget*ggml_tensor_overhead()), // opencoti RYS (#658): budget eff layers + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; +@@ -924,7 +954,14 @@ llama_kv_cache::llama_kv_cache( + + const bool is_mla = hparams.is_mla(); + +- for (uint32_t il = 0; il < hparams.n_layer; il++) { ++ // opencoti RYS (#658) — iterate the EFFECTIVE plan: `il = src = plan[eff]` selects the SOURCE ++ // layer whose dims/hparams size this slot (and tags layers[].il for later hparams re-derivation ++ // in the shift/defrag/state-IO loops, which index tensors directly, never re-keying map_layer_ids); ++ // `eff` is the distinct slot the graph's build_attn(eff) reaches via map_layer_ids[eff]. Tensor ++ // NAMES use `eff` so duplicated source layers get UNIQUE names (state-IO safety). EMPTY plan ⇒ ++ // eff==il==src for all layers ⇒ every size/name/map value is byte-identical to the shipped path. ++ for (uint32_t eff = 0; eff < n_layer_eff; eff++) { ++ const uint32_t il = layer_plan.empty() ? eff : (uint32_t) layer_plan[eff]; // SOURCE layer index + if (!hparams.has_kv(il)) { + LLAMA_LOG_DEBUG("%s: layer %3d: does not have KV cache\n", __func__, il); + continue; +@@ -957,7 +994,7 @@ llama_kv_cache::llama_kv_cache( + + LLAMA_LOG_WARN("%s: layer %3d: sharing with target layer %d\n", __func__, il, il_share); + +- map_layer_ids[il] = layers.size(); ++ map_layer_ids[eff] = layers.size(); // opencoti RYS (#658): key by eff (graph passes eff) + + layers.push_back(layer_share); + layers.back().il = il; +@@ -1108,16 +1145,16 @@ llama_kv_cache::llama_kv_cache( + // grep by name find an unchanged layout. + if (k_s) { + if (n_stream == 1) { +- ggml_format_name(k_s, "cache_k_l%d", il); ++ ggml_format_name(k_s, "cache_k_l%d", eff); // opencoti RYS (#658): name by eff + } else { +- ggml_format_name(k_s, "cache_k_l%d_s%u", il, s); ++ ggml_format_name(k_s, "cache_k_l%d_s%u", eff, s); // opencoti RYS (#658): name by eff + } + } + if (v_s) { + if (n_stream == 1) { +- ggml_format_name(v_s, "cache_v_l%d", il); ++ ggml_format_name(v_s, "cache_v_l%d", eff); // opencoti RYS (#658): name by eff + } else { +- ggml_format_name(v_s, "cache_v_l%d_s%u", il, s); ++ ggml_format_name(v_s, "cache_v_l%d_s%u", eff, s); // opencoti RYS (#658): name by eff + } + } + +@@ -1179,16 +1216,16 @@ llama_kv_cache::llama_kv_cache( + const char * v_base = layer_window ? "cache_v_tail" : "cache_v_cpu"; + if (k_cpu_s) { + if (n_stream == 1) { +- ggml_format_name(k_cpu_s, "%s_l%d", k_base, il); ++ ggml_format_name(k_cpu_s, "%s_l%d", k_base, eff); // opencoti RYS (#658): name by eff + } else { +- ggml_format_name(k_cpu_s, "%s_l%d_s%u", k_base, il, s); ++ ggml_format_name(k_cpu_s, "%s_l%d_s%u", k_base, eff, s); // opencoti RYS (#658): name by eff + } + } + if (v_cpu_s) { + if (n_stream == 1) { +- ggml_format_name(v_cpu_s, "%s_l%d", v_base, il); ++ ggml_format_name(v_cpu_s, "%s_l%d", v_base, eff); // opencoti RYS (#658): name by eff + } else { +- ggml_format_name(v_cpu_s, "%s_l%d_s%u", v_base, il, s); ++ ggml_format_name(v_cpu_s, "%s_l%d_s%u", v_base, eff, s); // opencoti RYS (#658): name by eff + } + } + k_cpu_per_stream.push_back(k_cpu_s); +@@ -1227,9 +1264,9 @@ llama_kv_cache::llama_kv_cache( + // [n_embd_k_gpu, n_block_sel, 2] f16 : [..,0]=min, [..,1]=max + ggml_tensor * kbounds_s = ggml_new_tensor_3d(ctx_s, GGML_TYPE_F16, n_embd_k_gpu, n_block_sel, 2); + if (n_stream == 1) { +- ggml_format_name(kbounds_s, "cache_kbounds_l%d", il); ++ ggml_format_name(kbounds_s, "cache_kbounds_l%d", eff); // opencoti RYS (#658): name by eff + } else { +- ggml_format_name(kbounds_s, "cache_kbounds_l%d_s%u", il, s); ++ ggml_format_name(kbounds_s, "cache_kbounds_l%d_s%u", eff, s); // opencoti RYS (#658): name by eff + } + kbounds_per_stream.push_back(kbounds_s); + +@@ -1239,13 +1276,13 @@ llama_kv_cache::llama_kv_cache( + // [n_block_sel] = reserved scratch. Init to [0,−1,…] post-alloc (see below). + if (n_stream == 1) { + ggml_tensor * block_sel_s = ggml_new_tensor_2d(ctx_s, GGML_TYPE_I32, n_block_sel + 1, 1); +- ggml_format_name(block_sel_s, "cache_block_sel_l%d", il); ++ ggml_format_name(block_sel_s, "cache_block_sel_l%d", eff); // opencoti RYS (#658): name by eff + block_sel_per_stream.push_back(block_sel_s); + } + } + } + +- map_layer_ids[il] = layers.size(); ++ map_layer_ids[eff] = layers.size(); // opencoti RYS (#658): key by eff (graph passes eff) + + layers.push_back({ + il, +@@ -1262,24 +1299,42 @@ llama_kv_cache::llama_kv_cache( + if (reuse) { + LLAMA_LOG_DEBUG("%s: reusing layers:\n", __func__); + +- for (uint32_t il = 0; il < hparams.n_layer; il++) { ++ // opencoti RYS (#665) — the reuse map (Gemma shared-KV globals: has_kv==false layers ++ // that read an earlier layer's cache) must be keyed by the EFFECTIVE slot the graph ++ // passes (eff), and must resolve its reuse TARGET (a SOURCE layer index) to a concrete ++ // eff slot too. Build src→first-eff so a duplicated shared-KV layer reads the earliest ++ // instance of the earlier layer's cache — a valid, deterministic superset. RYS off ⇒ ++ // eff==il==src ⇒ this is byte-identical to the upstream `map[il]=map[il_reuse]`. ++ std::map src_to_first_eff; ++ for (uint32_t eff = 0; eff < n_layer_eff; eff++) { ++ const uint32_t src = layer_plan.empty() ? eff : (uint32_t) layer_plan[eff]; ++ src_to_first_eff.emplace(src, eff); // emplace keeps the FIRST eff for each source ++ } ++ ++ for (uint32_t eff = 0; eff < n_layer_eff; eff++) { ++ const uint32_t il = layer_plan.empty() ? eff : (uint32_t) layer_plan[eff]; // SOURCE layer + const int32_t il_reuse = reuse(il); + + if (il_reuse < 0) { +- LLAMA_LOG_DEBUG("%s: - layer %3d: no reuse\n", __func__, il); ++ LLAMA_LOG_DEBUG("%s: - eff %3d (src %3d): no reuse\n", __func__, eff, il); + continue; + } + + if (filter && !filter(il)) { +- LLAMA_LOG_DEBUG("%s: - layer %3d: filtered\n", __func__, il); ++ LLAMA_LOG_DEBUG("%s: - eff %3d (src %3d): filtered\n", __func__, eff, il); + continue; + } + +- GGML_ASSERT(map_layer_ids.find(il_reuse) != map_layer_ids.end()); ++ const auto it_re = src_to_first_eff.find((uint32_t) il_reuse); ++ GGML_ASSERT(it_re != src_to_first_eff.end()); ++ const uint32_t eff_reuse = it_re->second; ++ ++ GGML_ASSERT(map_layer_ids.find(eff_reuse) != map_layer_ids.end()); + +- map_layer_ids[il] = map_layer_ids[il_reuse]; ++ map_layer_ids[eff] = map_layer_ids[eff_reuse]; + +- LLAMA_LOG_DEBUG("%s: - layer %3d: reuse layer %d, is_swa = %d\n", __func__, il, il_reuse, hparams.is_swa(il)); ++ LLAMA_LOG_DEBUG("%s: - eff %3d (src %3d): reuse src %d (eff %d), is_swa = %d\n", ++ __func__, eff, il, il_reuse, eff_reuse, hparams.is_swa(il)); + } + } + +@@ -2889,6 +2944,13 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k + + const auto & layer = layers[ikv]; + ++ // opencoti RYS (#660): map_layer_ids is keyed by the EFFECTIVE layer (the graph passes eff, ++ // which can exceed n_layer when --repeat-layers duplicates layers). Every hparams query below ++ // is a MODEL property that must be keyed by the SOURCE layer instead. Re-point il at the slot's ++ // source (layer.il) now that the cache tensor is resolved via ikv. RYS off ⇒ layer.il == il ⇒ ++ // byte-identical no-op. See docs/features/rys_layer_duplication.md §6. ++ il = layer.il; ++ + const uint64_t kv_size = get_size(); + + // opencoti F5 M2 headinfer — reassemble the full head set from the +@@ -3018,6 +3080,8 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k + + const auto & layer = layers[ikv]; + ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. ++ + const uint64_t kv_size = get_size(); + + // opencoti F5 M2 headinfer — concat the GPU + host head subsets. The +@@ -3541,6 +3605,7 @@ uint32_t llama_kv_cache::headinfer_gpu_heads(int32_t il) const { + ggml_tensor * llama_kv_cache::get_k_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.gpu_heads > 0 && "get_k_gpu called without an active headinfer split"); + + // opencoti F4 M3 Phase 4-D: per-stream M2 GPU subset. In unified +@@ -3577,6 +3642,7 @@ ggml_tensor * llama_kv_cache::get_kbounds(ggml_context * ctx, int32_t il, const + GGML_UNUSED(ctx); + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + if (layer.kbounds_per_stream.empty()) { + return nullptr; + } +@@ -3591,6 +3657,7 @@ ggml_tensor * llama_kv_cache::get_block_sel(ggml_context * ctx, int32_t il, cons + GGML_UNUSED(ctx); + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + if (layer.block_sel_per_stream.empty()) { + return nullptr; + } +@@ -3600,6 +3667,7 @@ ggml_tensor * llama_kv_cache::get_block_sel(ggml_context * ctx, int32_t il, cons + ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.gpu_heads > 0 + && !layer.k_cpu_per_stream.empty() + && layer.k_cpu_per_stream[0] +@@ -3644,6 +3712,7 @@ ggml_tensor * llama_kv_cache::get_k_cpu(ggml_context * ctx, int32_t il, uint32_t + ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.gpu_heads > 0 && "get_v_gpu called without an active headinfer split"); + // M2 requires non-transposed V (FA on). The split-active branch in + // get_v/cpy_v also requires this. Per-stream view + concat on stream +@@ -3675,6 +3744,7 @@ ggml_tensor * llama_kv_cache::get_v_gpu(ggml_context * ctx, int32_t il, uint32_t + ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.gpu_heads > 0 + && !layer.v_cpu_per_stream.empty() + && layer.v_cpu_per_stream[0] +@@ -3715,6 +3785,7 @@ ggml_tensor * llama_kv_cache::get_v_cpu(ggml_context * ctx, int32_t il, uint32_t + ggml_tensor * llama_kv_cache::get_k_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.window_cells > 0 && "get_k_window called without an active position window"); + + const uint32_t n_embd_head_k = hparams.n_embd_head_k(il); +@@ -3743,6 +3814,7 @@ ggml_tensor * llama_kv_cache::get_k_window(ggml_context * ctx, int32_t il, uint3 + ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.window_cells > 0 + && !layer.k_cpu_per_stream.empty() + && layer.k_cpu_per_stream[0] +@@ -3800,6 +3872,7 @@ ggml_tensor * llama_kv_cache::get_k_tail(ggml_context * ctx, int32_t il, uint32_ + ggml_tensor * llama_kv_cache::get_v_window(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.window_cells > 0 && "get_v_window called without an active position window"); + GGML_ASSERT(!v_trans && "position window requires non-transposed V"); + +@@ -3829,6 +3902,7 @@ ggml_tensor * llama_kv_cache::get_v_window(ggml_context * ctx, int32_t il, uint3 + ggml_tensor * llama_kv_cache::get_v_tail(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + const auto & layer = layers[ikv]; ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. + GGML_ASSERT(layer.window_cells > 0 + && !layer.v_cpu_per_stream.empty() + && layer.v_cpu_per_stream[0] +@@ -3885,6 +3959,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + + const auto & layer = layers[ikv]; + ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. ++ + const int64_t n_embd_head = k_cur->ne[0]; + const int64_t n_head = k_cur->ne[1]; + const int64_t n_tokens = k_cur->ne[2]; +@@ -4068,6 +4144,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + + const auto & layer = layers[ikv]; + ++ il = layer.il; // opencoti RYS (#660): hparams by SOURCE layer, not the eff map-key. See get_k. ++ + const int64_t n_embd_head = v_cur->ne[0]; + const int64_t n_head = v_cur->ne[1]; + const int64_t n_tokens = v_cur->ne[2]; +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +index f40eca7ef..497e9dcc1 100644 +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -165,7 +165,15 @@ public: + // history is low-bit, all resident. Consulted only in position-window mode + // (the tail tensor exists); ignored for a vanilla / M2-head-split cache. + ggml_type type_k_tail = GGML_TYPE_COUNT, +- ggml_type type_v_tail = GGML_TYPE_COUNT); ++ ggml_type type_v_tail = GGML_TYPE_COUNT, ++ // opencoti RYS (#658) — runtime weight-shared layer duplication. Effective ++ // execution plan: layer_plan[eff] = SOURCE transformer-layer index. When ++ // NON-EMPTY the cache allocates one distinct KV slot per EFF position (sized ++ // from its src's dims) and keys map_layer_ids by eff, so the graph's build_attn(eff) ++ // reaches the right slot; layers[].il stays = src for hparams re-derivation. ++ // EMPTY (every non-dense-Qwen caller / RYS off) = identity = byte-identical. ++ // See docs/features/rys_layer_duplication.md. ++ const std::vector & layer_plan = {}); + + ~llama_kv_cache() = default; + +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +index f618219ac..0a7e2cee5 100644 +--- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp +@@ -35,7 +35,9 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + bool kv_window_measure_pass, + /* layer filters */ + const layer_filter_cb & filter_attn, +- const layer_filter_cb & filter_recr) : ++ const layer_filter_cb & filter_recr, ++ /* opencoti RYS (#666) — see header */ ++ const std::vector & layer_plan) : + hparams(model.hparams), + mem_attn(new llama_kv_cache_iswa( + model, +@@ -75,7 +77,15 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + nullptr, + // opencoti bug-858 dual-context MTP — hybrid-iswa attn cache is standalone: no sharing. + /* mem_other */ nullptr, +- /* share */ nullptr ++ /* share */ nullptr, ++ // opencoti #551 sparse-attn — off for the hybrid-iswa attn cache. ++ /* sparse_attn_block_size */ 0, ++ // opencoti #582-P2 mixed-KV — uniform (no distinct tail type). ++ /* type_k_tail */ GGML_TYPE_COUNT, ++ /* type_v_tail */ GGML_TYPE_COUNT, ++ // opencoti RYS (#666) — forward the layer plan so a duplicated attention ++ // layer in a hybrid-iswa model gets its own KV slot (eff-keyed). ++ layer_plan + )), + mem_recr(new llama_memory_recurrent( + model, +@@ -87,7 +97,10 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa( + n_rs_seq, + filter_recr == nullptr ? + [&](int32_t il) { return hparams.is_recurrent(il); } +- : filter_recr ++ : filter_recr, ++ // opencoti RYS (#666) — forward the layer plan so a duplicated recurrent ++ // (gated-delta-net) layer gets its own state slot (eff-keyed). ++ layer_plan + )) {} + + llama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { +diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.h b/llama.cpp/src/llama-memory-hybrid-iswa.h +index 29666880f..d352a737e 100644 +--- a/llama.cpp/src/llama-memory-hybrid-iswa.h ++++ b/llama.cpp/src/llama-memory-hybrid-iswa.h +@@ -49,7 +49,13 @@ public: + bool kv_window_measure_pass = false, + /* layer filters */ + const layer_filter_cb & filter_attn = nullptr, +- const layer_filter_cb & filter_recr = nullptr); ++ const layer_filter_cb & filter_recr = nullptr, ++ /* opencoti RYS (#666) — weight-shared layer duplication ++ plan (eff→src). Forwarded to BOTH the inner iSWA attention ++ cache and the inner llama_memory_recurrent so a duplicated ++ layer gets its own attn-KV / recurrent-state slot. Empty = ++ identity = byte-identical (default). */ ++ const std::vector & layer_plan = {}); + + ~llama_memory_hybrid_iswa() = default; + +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +index 6c2272443..fa336c2c5 100644 +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -35,7 +35,9 @@ llama_memory_hybrid::llama_memory_hybrid( + bool kv_window_measure_pass, + /* layer filters */ + const layer_filter_cb & filter_attn, +- const layer_filter_cb & filter_recr) : ++ const layer_filter_cb & filter_recr, ++ /* opencoti RYS (#666) — see header */ ++ const std::vector & layer_plan) : + hparams(model.hparams), + mem_attn(new llama_kv_cache( + model, +@@ -75,7 +77,15 @@ llama_memory_hybrid::llama_memory_hybrid( + nullptr, + // opencoti bug-858 dual-context MTP — hybrid attn cache is standalone: no sharing. + /* mem_other */ nullptr, +- /* share */ nullptr ++ /* share */ nullptr, ++ // opencoti #551 sparse-attn — off for the hybrid attn cache. ++ /* sparse_attn_block_size */ 0, ++ // opencoti #582-P2 mixed-KV — uniform (no distinct tail type). ++ /* type_k_tail */ GGML_TYPE_COUNT, ++ /* type_v_tail */ GGML_TYPE_COUNT, ++ // opencoti RYS (#666) — forward the layer plan so a duplicated attention ++ // layer in a hybrid model gets its own KV slot (eff-keyed). ++ layer_plan + )), + mem_recr(new llama_memory_recurrent( + model, +@@ -87,7 +97,10 @@ llama_memory_hybrid::llama_memory_hybrid( + n_rs_seq, + filter_recr == nullptr ? + [&](int32_t il) { return hparams.is_recurrent(il); } +- : filter_recr ++ : filter_recr, ++ // opencoti RYS (#666) — forward the layer plan so a duplicated recurrent ++ // (gated-delta-net) layer gets its own state slot (eff-keyed). ++ layer_plan + )) {} + + llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { +diff --git a/llama.cpp/src/llama-memory-hybrid.h b/llama.cpp/src/llama-memory-hybrid.h +index b7540c52a..e41ba4fed 100644 +--- a/llama.cpp/src/llama-memory-hybrid.h ++++ b/llama.cpp/src/llama-memory-hybrid.h +@@ -49,7 +49,13 @@ public: + bool kv_window_measure_pass = false, + /* layer filters */ + const layer_filter_cb & filter_attn = nullptr, +- const layer_filter_cb & filter_recr = nullptr); ++ const layer_filter_cb & filter_recr = nullptr, ++ /* opencoti RYS (#666) — weight-shared layer duplication ++ plan (eff→src). Forwarded to BOTH the inner attention ++ llama_kv_cache and the inner llama_memory_recurrent so ++ a duplicated layer gets its own attn-KV / recurrent-state ++ slot. Empty = identity = byte-identical (default). */ ++ const std::vector & layer_plan = {}); + + ~llama_memory_hybrid() = default; + +diff --git a/llama.cpp/src/llama-memory-recurrent.cpp b/llama.cpp/src/llama-memory-recurrent.cpp +index ec5dc5835..38d2dbaf1 100644 +--- a/llama.cpp/src/llama-memory-recurrent.cpp ++++ b/llama.cpp/src/llama-memory-recurrent.cpp +@@ -25,8 +25,12 @@ llama_memory_recurrent::llama_memory_recurrent( + uint32_t mem_size, + uint32_t n_seq_max, + uint32_t n_rs_seq, +- const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) { ++ const layer_filter_cb & filter, ++ const std::vector & layer_plan) : hparams(model.hparams), n_seq_max(n_seq_max) { + const int32_t n_layer = hparams.n_layer; ++ // opencoti RYS (#666): effective layer count. Empty plan = identity (n_layer_eff == n_layer), ++ // and every eff==src below, so the state cache is byte-identical to upstream when RYS is off. ++ const int32_t n_layer_eff = layer_plan.empty() ? n_layer : (int32_t) layer_plan.size(); + + head = 0; + size = mem_size; +@@ -51,7 +55,7 @@ llama_memory_recurrent::llama_memory_recurrent( + auto it = ctx_map.find(buft); + if (it == ctx_map.end()) { + ggml_init_params params = { +- /*.mem_size =*/ size_t(2u*n_layer*ggml_tensor_overhead()), ++ /*.mem_size =*/ size_t(2u*n_layer_eff*ggml_tensor_overhead()), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; +@@ -69,12 +73,18 @@ llama_memory_recurrent::llama_memory_recurrent( + return it->second.get(); + }; + +- r_l.resize(n_layer); +- s_l.resize(n_layer); ++ r_l.resize(n_layer_eff); ++ s_l.resize(n_layer_eff); + +- for (int i = 0; i < n_layer; i++) { +- if (filter && !filter(i)) { +- LLAMA_LOG_DEBUG("%s: layer %3d: skipped\n", __func__, i); ++ // opencoti RYS (#666): iterate the EFFECTIVE plan. eff = state slot (indexes r_l/s_l and is what ++ // get_r_l/get_s_l receive from the graph); src = the SOURCE layer whose weights/device this pass ++ // re-runs (filter + dev_layer are MODEL properties keyed by src). State dims n_embd_r()/n_embd_s() ++ // are global, so every duplicated recurrent layer gets an independent, uniform-shape state slot. ++ for (int eff = 0; eff < n_layer_eff; eff++) { ++ const int src = layer_plan.empty() ? eff : layer_plan[eff]; ++ ++ if (filter && !filter(src)) { ++ LLAMA_LOG_DEBUG("%s: layer %3d (src %3d): skipped\n", __func__, eff, src); + continue; + } + +@@ -83,13 +93,13 @@ llama_memory_recurrent::llama_memory_recurrent( + ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type(); + + if (offload) { +- auto * dev = model.dev_layer(i); ++ auto * dev = model.dev_layer(src); + buft = ggml_backend_dev_buffer_type(dev); + + dev_name = ggml_backend_dev_name(dev); + } + +- LLAMA_LOG_DEBUG("%s, layer %3d: dev = %s\n", __func__, i, dev_name); ++ LLAMA_LOG_DEBUG("%s, layer %3d (src %3d): dev = %s\n", __func__, eff, src, dev_name); + + ggml_context * ctx = ctx_for_buft(buft); + if (!ctx) { +@@ -99,10 +109,10 @@ llama_memory_recurrent::llama_memory_recurrent( + const uint32_t n_rows = mem_size * (1 + n_rs_seq); + ggml_tensor * r = ggml_new_tensor_2d(ctx, type_r, hparams.n_embd_r(), n_rows); + ggml_tensor * s = ggml_new_tensor_2d(ctx, type_s, hparams.n_embd_s(), n_rows); +- ggml_format_name(r, "cache_r_l%d", i); +- ggml_format_name(s, "cache_s_l%d", i); +- r_l[i] = r; +- s_l[i] = s; ++ ggml_format_name(r, "cache_r_l%d", eff); ++ ggml_format_name(s, "cache_s_l%d", eff); ++ r_l[eff] = r; ++ s_l[eff] = s; + } + + // allocate tensors and initialize the buffers to avoid NaNs in the padding +diff --git a/llama.cpp/src/llama-memory-recurrent.h b/llama.cpp/src/llama-memory-recurrent.h +index b13b7b748..374edf1c4 100644 +--- a/llama.cpp/src/llama-memory-recurrent.h ++++ b/llama.cpp/src/llama-memory-recurrent.h +@@ -24,7 +24,13 @@ public: + uint32_t mem_size, + uint32_t n_seq_max, + uint32_t n_rs_seq, +- const layer_filter_cb & filter); ++ const layer_filter_cb & filter, ++ // opencoti RYS (#666) — runtime weight-shared layer duplication plan ++ // (eff→src). r_l/s_l are sized n_layer_eff and indexed by eff so each ++ // duplicated recurrent (gated-delta-net) layer gets its OWN state slot. ++ // Empty = identity = byte-identical (default). See ++ // docs/features/rys_layer_duplication.md. ++ const std::vector & layer_plan = {}); + + ~llama_memory_recurrent() = default; + +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +index e33d9aee4..29f620e63 100644 +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -1991,7 +1991,10 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + std::max((uint32_t) 1, cparams.n_seq_max), + cparams.n_seq_max, + cparams.n_rs_seq, +- nullptr); ++ nullptr, ++ // opencoti RYS (#666) — weight-shared layer duplication plan ++ // (empty = byte-identical). See docs/features/rys_layer_duplication.md. ++ cparams.layer_plan); + } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen35) { + // The main difference between hybrid architectures is the + // layer filters, so pick the right one here +@@ -2045,7 +2048,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + /* kv_compute_reserve_mib */ cparams.kv_compute_reserve_mib, + /* kv_window_measure_pass */ cparams.kv_window_measure_pass, + /* filter_attn */ std::move(filter_attn), +- /* filter_recr */ std::move(filter_recr)); ++ /* filter_recr */ std::move(filter_recr), ++ // opencoti RYS (#666) — weight-shared layer duplication plan. ++ /* layer_plan */ cparams.layer_plan); + } else { + res = new llama_memory_hybrid( + /* model */ *this, +@@ -2073,7 +2078,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + /* kv_compute_reserve_mib */ cparams.kv_compute_reserve_mib, + /* kv_window_measure_pass */ cparams.kv_window_measure_pass, + /* filter_attn */ std::move(filter_attn), +- /* filter_recr */ std::move(filter_recr)); ++ /* filter_recr */ std::move(filter_recr), ++ // opencoti RYS (#666) — weight-shared layer duplication plan. ++ /* layer_plan */ cparams.layer_plan); + } + } else { + llama_memory_i::layer_reuse_cb reuse = nullptr; +@@ -2155,7 +2162,12 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + // opencoti #582-P2 — distinct spilled-tail KV type + // (GGML_TYPE_COUNT = same as window → byte-identical). + cparams.type_k_tail, +- cparams.type_v_tail); ++ cparams.type_v_tail, ++ // opencoti RYS (#665) — runtime weight-shared layer duplication. ++ // iSWA path (Gemma-4 dense 12B/31B + A4B, E-series). Empty plan ++ // (no --repeat-layers) = identity = byte-identical. Forwarded to ++ // both base/swa inner caches; reuse loop is eff-aware. ++ cparams.layer_plan); + } else { + GGML_ASSERT(!hparams.is_swa_any()); + +@@ -2198,7 +2210,13 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, + // opencoti #582-P2 — distinct spilled-tail KV type + // (GGML_TYPE_COUNT = same as window → byte-identical). + cparams.type_k_tail, +- cparams.type_v_tail); ++ cparams.type_v_tail, ++ // opencoti RYS (#658) — runtime weight-shared layer duplication. ++ // Dense full-attention path only (the RYS MVP target: Qwen3/3.5/3.6 ++ // dense). Empty plan (no --repeat-layers) = identity = byte-identical. ++ // The iswa/hybrid caches keep the empty default until RYS Phase 2; ++ // RYS-4 hard-errors --repeat-layers on those archs. ++ cparams.layer_plan); + } + } + } +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +index 3703ec154..0d3353583 100644 +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -191,7 +191,17 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer); + } + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti-hook: rys-layer-dup (#665) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication on the iSWA path. Iterate the EFFECTIVE plan: ++ // `src = plan[eff]` selects the source layer's WEIGHTS / hparams / per-layer-embd / is_swa ++ // (a MODEL property — a duplicated layer keeps its source's SWA-ness, head_dim 256/512, rope ++ // regime, MoE-ness); `eff` is the distinct KV-slot index handed to build_attn / build_attn_dca ++ // (the iSWA base/swa inner caches are eff-length, keyed by eff, partitioned by is_swa(src)). ++ // Empty plan == OFF == identity (eff==src==il) == byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + const int64_t n_embd_head = hparams.n_embd_head_k(il); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); + +@@ -263,30 +273,35 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + + if (dca_l) { // bug-2118 (wo_s dropped like the qwen3 DCA path — null on these GGUFs) + const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors }; ++ // opencoti-hook: rys-layer-dup (#665) — weights by `il`(src), KV slot by `eff`. + cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr, +- Qcur, Kcur, Vcur, rp, hparams.f_attention_scale, il); ++ Qcur, Kcur, Vcur, rp, hparams.f_attention_scale, eff); + } else { ++ // opencoti-hook: rys-layer-dup (#665) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, model.layers[il].wo, + nullptr, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, +- hparams.f_attention_scale, il); ++ hparams.f_attention_scale, eff); + } + } else if (dca_l) { + // bug-2118: shared-KV global layer under DCA — Q-only read of the earlier + // layer's DCA-formatted cache (null k/v -> no store). + const dca_rope rp = { freq_base_l, freq_scale_l, n_rot_l, freq_factors }; ++ // opencoti-hook: rys-layer-dup (#665) — weights by `il`(src), KV slot by `eff` ++ // (the eff-aware reuse map routes this shared-KV slot to the earlier layer's cache). + cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, nullptr, +- Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, il); ++ Qcur, nullptr, nullptr, rp, hparams.f_attention_scale, eff); + } else { + // reuse KV cache of earlier layers ++ // opencoti-hook: rys-layer-dup (#665) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, + model.layers[il].wo, nullptr, model.layers[il].wo_s, +- Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); ++ Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, eff); + } + + // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing + // opencoti bug-858: skip the early strip for the MTP target (mtp_defer_out_ids) so the last + // layer runs on all n_tokens rows and t_h_pre_norm below stays ungathered. +- if (il == n_layer - 1 && inp_out_ids && !mtp_defer_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids && !mtp_defer_out_ids) { // opencoti RYS (#665): last EFF pass + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } +@@ -388,7 +403,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para + // TODO @ngxson : improve this + // opencoti bug-858: same deferred strip as the main path (keep per-layer input full for + // the MTP target so its last-layer residual matches the ungathered hidden). +- if (il == n_layer - 1 && inp_out_ids && !mtp_defer_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids && !mtp_defer_out_ids) { // opencoti RYS (#665): last EFF pass + inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids); + } + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +index 6efae4a2d..bfe148246 100644 +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -1715,16 +1715,20 @@ struct llama_model_qwen3next : public llama_model_base { + struct graph : public llm_build_delta_net_base { + graph(const llama_model & model, const llm_graph_params & params); + private: ++ // opencoti RYS (#666): il = SOURCE layer (weights/arch); il_kv = distinct KV/state slot ++ // (== il when RYS off). See docs/features/rys_layer_duplication.md. + ggml_tensor * build_layer_attn( + llm_graph_input_attn_kv * inp_attn, + ggml_tensor * cur, + ggml_tensor * inp_pos, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_ffn( + ggml_tensor * cur, +@@ -1756,18 +1760,22 @@ struct llama_model_qwen35 : public llama_model_base { + struct graph : public llm_build_delta_net_base { + graph(const llama_model & model, const llm_graph_params & params); + private: ++ // opencoti RYS (#657): il = SOURCE layer (weights/arch); il_kv = distinct KV/state slot ++ // (== il when RYS off). See docs/features/rys_layer_duplication.md. + ggml_tensor * build_layer_attn( + llm_graph_input_attn_kv * inp_attn, + llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off, unchanged) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_ffn( + ggml_tensor * cur, +@@ -1803,18 +1811,22 @@ struct llama_model_qwen35moe : public llama_model_base { + struct graph : public llm_build_delta_net_base { + graph(const llama_model & model, const llm_graph_params & params); + private: ++ // opencoti RYS (#666): il = SOURCE layer (weights/arch); il_kv = distinct KV/state slot ++ // (eff). il_kv == il when RYS off ⇒ byte-identical. + ggml_tensor * build_layer_attn( + llm_graph_input_attn_kv * inp_attn, + llm_graph_input_dca * inp_dca, // opencoti F5 dca #623 (nullptr => --dca off, unchanged) + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il); ++ int il, ++ int il_kv); + + ggml_tensor * build_layer_ffn( + ggml_tensor * cur, +diff --git a/llama.cpp/src/models/qwen2.cpp b/llama.cpp/src/models/qwen2.cpp +index 397eddf3a..8a4687b36 100644 +--- a/llama.cpp/src/models/qwen2.cpp ++++ b/llama.cpp/src/models/qwen2.cpp +@@ -73,7 +73,15 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti-hook: rys-layer-dup (#657) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication: iterate the EFFECTIVE plan instead of the ++ // identity [0,n_layer). `src = plan[eff]` selects which source layer's WEIGHTS/arch to run ++ // (shared by pointer, zero extra param VRAM); `eff` is the distinct KV-slot index handed to ++ // build_attn. Empty plan == OFF == identity (eff==src==il) == byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + ggml_tensor * inpSA = inpL; + + // norm +@@ -110,15 +118,17 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param + + if (use_dca) { // opencoti F5 dca #593 + const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, +- Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), eff); + } else { ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), eff); + } + } +- if (il == n_layer - 1 && inp_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +diff --git a/llama.cpp/src/models/qwen2moe.cpp b/llama.cpp/src/models/qwen2moe.cpp +index 7cb03859d..0442bf5bb 100644 +--- a/llama.cpp/src/models/qwen2moe.cpp ++++ b/llama.cpp/src/models/qwen2moe.cpp +@@ -79,7 +79,15 @@ llama_model_qwen2moe::graph::graph(const llama_model & model, const llm_graph_pa + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti-hook: rys-layer-dup (#657) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication: iterate the EFFECTIVE plan instead of the ++ // identity [0,n_layer). `src = plan[eff]` selects which source layer's WEIGHTS/arch to run ++ // (shared by pointer, zero extra param VRAM); `eff` is the distinct KV-slot index handed to ++ // build_attn. Empty plan == OFF == identity (eff==src==il) == byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + ggml_tensor * inpSA = inpL; + + // norm +@@ -110,11 +118,12 @@ llama_model_qwen2moe::graph::graph(const llama_model & model, const llm_graph_pa + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), eff); + } +- if (il == n_layer - 1 && inp_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +diff --git a/llama.cpp/src/models/qwen3.cpp b/llama.cpp/src/models/qwen3.cpp +index 54697daac..4258635af 100644 +--- a/llama.cpp/src/models/qwen3.cpp ++++ b/llama.cpp/src/models/qwen3.cpp +@@ -73,7 +73,17 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti-hook: rys-layer-dup (#657) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication: iterate the EFFECTIVE plan instead of the ++ // identity [0,n_layer). `src = plan[eff]` selects which source layer's WEIGHTS/arch to run ++ // (shared by pointer, zero extra param VRAM); `eff` is the distinct KV-slot index handed to ++ // build_attn (each duplicated pass writes its own K/V at the same token positions). Empty plan ++ // == OFF == identity (eff==src==il) == byte-identical. On-path needs the eff-length KV alloc ++ // from RYS-3 (#658); until then a non-empty plan is hard-errored at load. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + ggml_tensor * inpSA = inpL; + + // norm +@@ -116,15 +126,17 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param + + if (use_dca) { // opencoti F5 dca #593 + const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, +- Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), eff); + } else { ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), eff); + } + } +- if (il == n_layer - 1 && inp_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp +index 827779e84..762882fc8 100644 +--- a/llama.cpp/src/models/qwen35.cpp ++++ b/llama.cpp/src/models/qwen35.cpp +@@ -166,7 +166,18 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + + // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. + const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers; +- for (int il = 0; il < n_transformer_layers; ++il) { ++ ++ // opencoti-hook: rys-layer-dup (#657) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication on the hybrid (full-attn + gated-delta-net) ++ // stack: iterate the EFFECTIVE plan; `il = src = plan[eff]` drives WEIGHTS + arch props ++ // (is_recurrent, rope, norms — shared by pointer, zero extra param VRAM); `eff` is the ++ // distinct KV / recurrent-state slot (each duplicated pass keeps its own K/V + conv/ssm ++ // state at the same token positions). Empty plan == OFF == identity (eff==src==il) == ++ // byte-identical. On-path needs the eff-length KV+RS alloc from RYS-3 (#658). ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_transformer_layers : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); +@@ -174,16 +185,17 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para + + ggml_build_forward_expand(gf, cur); + +- // Determine layer type and build appropriate attention mechanism ++ // Determine layer type and build appropriate attention mechanism. `il`(src) selects the ++ // layer TYPE + weights; `eff` selects the distinct KV / recurrent-state slot. + if (hparams.is_recurrent(il)) { + // Linear attention layer (gated delta net) +- cur = build_layer_attn_linear(inp->get_recr(), cur, il); ++ cur = build_layer_attn_linear(inp->get_recr(), cur, il, eff); + } else { + // Full attention layer +- cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il); ++ cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il, eff); + } + +- if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { ++ if (eff == n_layer_eff - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +@@ -281,7 +293,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +- int il) { ++ int il, // SOURCE layer — weights + arch props ++ int il_kv) { // opencoti RYS (#657) — distinct KV slot (== il when RYS off) + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + +@@ -332,7 +345,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + // NEOX-collapse produced garbage (gate diverged @char 21, #623). DCA needs an f16 K cache (default). + const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr, + { sections[0], sections[1], sections[2], sections[3] } }; +- cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il); ++ cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il_kv); // opencoti RYS (#657): KV slot by eff + } else { + // Apply MRoPE + Qcur = ggml_rope_multi( +@@ -353,7 +366,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + + cur = build_attn(inp, + nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il_kv); // opencoti RYS (#657): KV slot by eff + } + cb(cur, "attn_pregate", il); + +@@ -372,7 +385,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( + ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il) { ++ int il, // SOURCE layer — weights + arch props ++ int il_kv) { // opencoti RYS (#657) — distinct recurrent-state slot (== il when RYS off) + const auto * mctx_cur = inp->mctx; + + const int64_t d_inner = hparams.ssm_d_inner; +@@ -412,14 +426,14 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( + + gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs); + +- ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); +- ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); ++ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il_kv); // opencoti RYS (#657): recurrent-state slot by eff ++ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il_kv); // opencoti RYS (#657): recurrent-state slot by eff + + ggml_tensor * conv_kernel = model.layers[il].ssm_conv1d; + const int64_t conv_kernel_size = conv_kernel->ne[0]; + const int64_t conv_channels = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state; + +- ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il); ++ ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il_kv); // opencoti RYS (#657): state slot by eff + + ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs); +@@ -481,7 +495,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( + cb(k_conv, "k_conv_predelta", il); + cb(v_conv, "v_conv_predelta", il); + +- ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); ++ ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il_kv); // opencoti RYS (#657): state slot by eff + + // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim] + ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); +diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp +index af9ea1e77..ea2952380 100644 +--- a/llama.cpp/src/models/qwen35moe.cpp ++++ b/llama.cpp/src/models/qwen35moe.cpp +@@ -189,7 +189,12 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p + + // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. + const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers; +- for (int il = 0; il < n_transformer_layers; ++il) { ++ // opencoti RYS (#666) — plan-driven forward loop. eff = execution slot (KV/state slot); ++ // il = SOURCE layer (weights/hparams). == eff when RYS off ⇒ byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_transformer_layers : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer; == eff when RYS off + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); +@@ -200,13 +205,13 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p + // Determine layer type and build appropriate attention mechanism + if (hparams.is_recurrent(il)) { + // Linear attention layer (gated delta net) +- cur = build_layer_attn_linear(inp->get_recr(), cur, il); ++ cur = build_layer_attn_linear(inp->get_recr(), cur, il, eff); + } else { + // Full attention layer +- cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il); ++ cur = build_layer_attn(inp->get_attn(), inp_dca, cur, inp_pos, sections, il, eff); + } + +- if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { ++ if (eff == n_layer_eff - 1 && inp_out_ids && cparams.embeddings_pre_norm_masked) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +@@ -293,7 +298,8 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + ggml_tensor * cur, + ggml_tensor * inp_pos, + int * sections, +- int il) { ++ int il, ++ int il_kv) { // opencoti RYS (#666) — distinct KV slot (== il when RYS off) + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + +@@ -344,7 +350,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + // NEOX-collapse produced garbage (gate diverged @char 21, #623). DCA needs an f16 K cache (default). + const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr, + { sections[0], sections[1], sections[2], sections[3] } }; +- cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il); ++ cur = build_attn_dca(inp, inp_dca, nullptr, nullptr, Qcur, Kcur, Vcur, rp, kq_scale, il_kv); // opencoti RYS (#666): KV slot by eff + } else { + // Apply IMRoPE + Qcur = ggml_rope_multi( +@@ -365,7 +371,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + + cur = build_attn(inp, + nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il_kv); // opencoti RYS (#666): KV slot by eff + } + cb(cur, "attn_pregate", il); + +@@ -384,7 +390,8 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( + ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il) { ++ int il, ++ int il_kv) { // opencoti RYS (#666) — distinct recurrent-state slot (== il when RYS off) + const auto * mctx_cur = inp->mctx; + + const int64_t d_inner = hparams.ssm_d_inner; +@@ -424,14 +431,14 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( + + gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs); + +- ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); +- ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); ++ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il_kv); // opencoti RYS (#666): recurrent-state slot by eff ++ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il_kv); // opencoti RYS (#666): recurrent-state slot by eff + + ggml_tensor * conv_kernel = model.layers[il].ssm_conv1d; + const int64_t conv_kernel_size = conv_kernel->ne[0]; + const int64_t conv_channels = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state; + +- ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il); ++ ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il_kv); // opencoti RYS (#666): state slot by eff + + ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs); +@@ -493,7 +500,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( + cb(k_conv, "k_conv_predelta", il); + cb(v_conv, "v_conv_predelta", il); + +- ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); ++ ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il_kv); // opencoti RYS (#666): state slot by eff + + // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim] + ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); +diff --git a/llama.cpp/src/models/qwen3moe.cpp b/llama.cpp/src/models/qwen3moe.cpp +index d83287081..39f5d355f 100644 +--- a/llama.cpp/src/models/qwen3moe.cpp ++++ b/llama.cpp/src/models/qwen3moe.cpp +@@ -83,7 +83,15 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti-hook: rys-layer-dup (#657) — see docs/features/rys_layer_duplication.md. ++ // RYS runtime weight-shared layer duplication: iterate the EFFECTIVE plan instead of the ++ // identity [0,n_layer). `src = plan[eff]` selects which source layer's WEIGHTS/arch to run ++ // (shared by pointer, zero extra param VRAM); `eff` is the distinct KV-slot index handed to ++ // build_attn. Empty plan == OFF == identity (eff==src==il) == byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer (weights/arch); == eff when RYS off + ggml_tensor * inpSA = inpL; + + // norm +@@ -126,15 +134,17 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa + + if (use_dca) { // opencoti F5 dca #593 + const dca_rope rp = { (float) freq_base, (float) freq_scale, (int) n_rot, nullptr }; ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn_dca(inp_attn, inp_dca, model.layers[il].wo, model.layers[il].wo_b, +- Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, rp, 1.0f/sqrtf(float(n_embd_head)), eff); + } else { ++ // opencoti-hook: rys-layer-dup (#657) — weights by `il`(src), KV slot by `eff`. + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), eff); + } + } +- if (il == n_layer - 1 && inp_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +diff --git a/llama.cpp/src/models/qwen3next.cpp b/llama.cpp/src/models/qwen3next.cpp +index 1d873427d..7eb14ff38 100644 +--- a/llama.cpp/src/models/qwen3next.cpp ++++ b/llama.cpp/src/models/qwen3next.cpp +@@ -120,7 +120,12 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + +- for (int il = 0; il < n_layer; ++il) { ++ // opencoti RYS (#666) — plan-driven forward loop. eff = execution slot (KV/state slot); ++ // il = SOURCE layer (weights/hparams). == eff when RYS off ⇒ byte-identical. ++ const auto & rys_plan = cparams.layer_plan; ++ const int n_layer_eff = rys_plan.empty() ? n_layer : (int) rys_plan.size(); ++ for (int eff = 0; eff < n_layer_eff; ++eff) { ++ const int il = rys_plan.empty() ? eff : rys_plan[eff]; // SOURCE layer; == eff when RYS off + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); +@@ -131,13 +136,13 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p + // Determine layer type and build appropriate attention mechanism + if (hparams.is_recurrent(il)) { + // Linear attention layer (gated delta net) +- cur = build_layer_attn_linear(inp->get_recr(), cur, il); ++ cur = build_layer_attn_linear(inp->get_recr(), cur, il, eff); + } else { + // Full attention layer +- cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); ++ cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il, eff); + } + +- if (il == n_layer - 1 && inp_out_ids) { ++ if (eff == n_layer_eff - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } +@@ -207,7 +212,8 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( + llm_graph_input_attn_kv * inp, + ggml_tensor * cur, + ggml_tensor * inp_pos, +- int il) { ++ int il, ++ int il_kv) { // opencoti RYS (#666) — distinct KV slot (== il when RYS off) + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + +@@ -263,7 +269,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( + + cur = build_attn(inp, + nullptr, nullptr, nullptr, +- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); ++ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il_kv); // opencoti RYS (#666): KV slot by eff + cb(cur, "attn_pregate", il); + + // TODO: CUDA is missing non-contiguous unary ops. when implemented: remove this cont +@@ -367,7 +373,8 @@ std::pair llama_model_qwen3next::graph::build_qkvz + ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( + llm_graph_input_rs * inp, + ggml_tensor * cur, +- int il) { ++ int il, ++ int il_kv) { // opencoti RYS (#666) — distinct recurrent-state slot (== il when RYS off) + const auto * mctx_cur = inp->mctx; + + const int64_t d_inner = hparams.ssm_d_inner; +@@ -427,14 +434,14 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( + beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); + gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs); + +- ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); +- ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); ++ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il_kv); // opencoti RYS (#666): recurrent-state slot by eff ++ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il_kv); // opencoti RYS (#666): recurrent-state slot by eff + + ggml_tensor * conv_kernel = model.layers[il].ssm_conv1d; + const int64_t conv_kernel_size = conv_kernel->ne[0]; + const int64_t conv_channels = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state; + +- ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il); ++ ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il_kv); // opencoti RYS (#666): state slot by eff + + ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs); +@@ -511,7 +518,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( + cb(k_conv, "k_conv_predelta", il); + cb(v_conv, "v_conv_predelta", il); + +- ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); ++ ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il_kv); // opencoti RYS (#666): state slot by eff + + // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim] + ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); diff --git a/patches/0123-rys-probe.patch b/patches/0123-rys-probe.patch new file mode 100644 index 0000000000000000000000000000000000000000..fdb5c4c1ab92c08f20b7521a6b9d7eb3f971afe4 --- /dev/null +++ b/patches/0123-rys-probe.patch @@ -0,0 +1,488 @@ +opencoti patch 0123 — RYS template probe (--rys-probe): cheap ΔPPL + graded task-probe shortlist generator for --repeat-layers (2 templates + external-eval hook) + +Adds a new `--rys-probe` PROGRAM MODE that maps a GGUF for good RYS +layer-duplication templates without reproducing dnhkng RYS-II's days-long +beam+surrogate+graded-benchmark search. It loads the model ONCE +(common_init_from_params model_only), then for each contiguous mid-stack block +[i,j) in the boundary-safe band [max(3,n/8)..n-max(3,n/8)) rebuilds ONLY the +llama_context with a mutated repeat_layers plan (weights are plan-independent) +and scores it by (a) ΔPPL over a small corpus (-f/-p, else a built-in slice) +and (b) a graded built-in arithmetic/word-problem/factual/sequence task-probe +(greedy decode + substring-match early-stop; frac-correct used as rank signal +AND guardrail). It prints a ranked table then exactly two ready-to-paste +templates — MOST EFFICIENT (best ΔPPL per added layer) and MAX GAIN (largest +ΔPPL) — which the companion perf/llamafile/rys-probe-eval.sh then validates on a +real downstream eval per template. Host-only, no CUDA, no new kernels; a model +run without --rys-probe is byte-identical. New file llamafile/rys_probe.cpp +(driver, ProgramMode::PROBE), wired through llamafile/{args.h,args.cpp,main.cpp, +BUILD.mk}; four additive common_params fields + four --rys-probe* flags in +llama.cpp/common/{common.h,arg.cpp}. Companion to the shipped RYS execution +engine (patch 0122, --repeat-layers). See docs/features/rys_probe.md. + +diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk +index da2ab85..49c8979 100644 +--- a/llamafile/BUILD.mk ++++ b/llamafile/BUILD.mk +@@ -142,6 +142,7 @@ LLAMAFILE_SRCS_CPP := \ + llamafile/chatbot_logo.cpp \ + llamafile/chatbot_main.cpp \ + llamafile/chatbot_repl.cpp \ ++ llamafile/rys_probe.cpp \ + llamafile/compute.cpp \ + llamafile/datauri.cpp \ + llamafile/extract_data_uris.cpp \ +diff --git a/llamafile/args.cpp b/llamafile/args.cpp +index 2ec119e..1da1226 100644 +--- a/llamafile/args.cpp ++++ b/llamafile/args.cpp +@@ -61,7 +61,9 @@ LlamafileArgs parse_llamafile_args(int argc, char** argv) { + + // Determine execution mode from flags + // Priority: explicit flags override defaults +- if (llamafile_has(argv, "--server")) { ++ if (llamafile_has(argv, "--rys-probe")) { ++ args.mode = ProgramMode::PROBE; ++ } else if (llamafile_has(argv, "--server")) { + args.mode = ProgramMode::SERVER; + } else if (llamafile_has(argv, "--chat")) { + args.mode = ProgramMode::CHAT; +diff --git a/llamafile/args.h b/llamafile/args.h +index 86a63e2..6739152 100644 +--- a/llamafile/args.h ++++ b/llamafile/args.h +@@ -28,6 +28,7 @@ enum class ProgramMode { + CHAT, // --chat: TUI chat only + SERVER, // --server: HTTP server only + CLI, // --cli: Single prompt -> response, then exit ++ PROBE, // --rys-probe: RYS template probe, then exit (docs/features/rys_probe.md) + }; + + // Parsed llamafile arguments +diff --git a/llamafile/main.cpp b/llamafile/main.cpp +index c5fb79b..2707dde 100644 +--- a/llamafile/main.cpp ++++ b/llamafile/main.cpp +@@ -101,6 +101,11 @@ extern int server_main(int argc, char **argv, + std::function on_ready, + std::function)> on_shutdown_available); + ++namespace lf { ++// opencoti RYS probe (`--rys-probe`) — see llamafile/rys_probe.cpp, docs/features/rys_probe.md ++int rys_probe_main(int argc, char **argv); ++} // namespace lf ++ + static void print_general_help() { + printf("llamafile v" LLAMAFILE_VERSION_STRING " - run LLMs locally\n" + "\n" +@@ -383,6 +388,10 @@ int main(int argc, char **argv) { + // Single prompt -> response mode + return lf::chatbot::cli_main(args.llama_argc, args.llama_argv); + ++ case lf::ProgramMode::PROBE: ++ // RYS template probe mode (docs/features/rys_probe.md) ++ return lf::rys_probe_main(args.llama_argc, args.llama_argv); ++ + case lf::ProgramMode::AUTO: + // Combined mode: server on main thread, TUI as HTTP client on background thread + return lf::combined_main(args); +diff --git a/llamafile/rys_probe.cpp b/llamafile/rys_probe.cpp +new file mode 100644 +index 0000000..5064fbf +--- /dev/null ++++ b/llamafile/rys_probe.cpp +@@ -0,0 +1,395 @@ ++// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*- ++// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi ++// ++// opencoti RYS probe (`--rys-probe`) — see docs/features/rys_probe.md ++// ++// Shortlist generator for RYS layer-duplication templates (`--repeat-layers`). ++// Loads the model ONCE, then for each candidate contiguous mid-stack block ++// rebuilds only the llama_context with a mutated repeat_layers plan and scores ++// it by (a) ΔPPL on a small corpus and (b) a small arithmetic/short-reasoning ++// task-probe. Prints a ΔPPL-vs-overhead Pareto shortlist + the best ++// --repeat-layers spec to paste. Host-only; no CUDA, no new kernels. ++// ++// This is a *shortlist*, not a verdict: PPL is a weak proxy for reasoning gains ++// (dnhkng RYS-II used graded Math/EQ). Validate the top-K on the real harness. ++ ++#include "common.h" ++#include "arg.h" ++#include "log.h" ++#include "llama.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace lf { ++ ++namespace { ++ ++struct Candidate { ++ int i = 0, j = 0; // duplicate source layers [i, j) ++ std::string spec; // "i,j" (empty = base) ++ double overhead_pct = 0.0; // (j-i)/n_transformer * 100 ++ bool ok = false; ++ double ppl = 0.0; ++ double dppl = 0.0; // ppl - base_ppl (negative = better) ++ double task = 0.0; // fraction correct on the task-probe ++}; ++ ++struct TaskQA { const char * q; const char * a; }; ++ ++// Built-in graded probe: arithmetic, multi-step word problems, factual recall, ++// and sequence completion (base-model friendly, greedy, substring-graded). ~24 ++// items so the `task` score has real resolution (1/24 ≈ 0.04) to RANK templates, ++// not just flag a broken one. Greedy decode early-stops as soon as the answer ++// substring appears, so cost is a few tokens/item in practice. ++const TaskQA kTasks[] = { ++ {"Q: What is 17 + 26?\nA:", "43"}, ++ {"Q: What is 8 times 7?\nA:", "56"}, ++ {"Q: What is 100 minus 37?\nA:", "63"}, ++ {"Q: What is 144 divided by 12?\nA:", "12"}, ++ {"Q: What is 9 times 6?\nA:", "54"}, ++ {"Q: What is 15 plus 27?\nA:", "42"}, ++ {"Q: What is 81 divided by 9?\nA:", "9"}, ++ {"Q: What is 13 times 4?\nA:", "52"}, ++ {"Q: What is 250 minus 75?\nA:", "175"}, ++ {"Q: What is half of 96?\nA:", "48"}, ++ {"Q: What is 7 squared?\nA:", "49"}, ++ {"Q: What is 3 to the power of 4?\nA:", "81"}, ++ {"Q: If a train travels 60 km in 2 hours, its speed in km/h is\nA:", "30"}, ++ {"Q: A shop sells pens at 3 dollars each. Five pens cost how many dollars?\nA:", "15"}, ++ {"Q: Tom has 12 apples and gives away 5. How many are left?\nA:", "7"}, ++ {"Q: There are 24 hours in a day. How many hours are in 3 days?\nA:", "72"}, ++ {"Q: The capital of France is\nA:", "Paris"}, ++ {"Q: The capital of Japan is\nA:", "Tokyo"}, ++ {"Q: The chemical symbol for water is\nA:", "H2O"}, ++ {"Q: The largest planet in the solar system is\nA:", "Jupiter"}, ++ {"Q: The opposite of hot is\nA:", "cold"}, ++ {"Q: Complete the sequence: 2, 4, 6, 8,\nA:", "10"}, ++ {"Q: Complete the sequence: 1, 2, 4, 8,\nA:", "16"}, ++ {"Q: How many sides does a triangle have?\nA:", "3"}, ++}; ++ ++// A tiny default corpus used when the user passes no -f/-p (enough tokens for a ++// stable-enough PPL delta on a small model; the user should pass -f for rigor). ++const char * kDefaultCorpus = ++ "The history of computing spans many centuries. Early mechanical calculators " ++ "gave way to electronic machines, and then to the stored-program computer. " ++ "Language models predict the next token in a sequence given the tokens that " ++ "came before it. Attention lets each position gather information from every " ++ "other position. Perplexity measures how well a model predicts a held-out " ++ "sample: lower is better. A merged or self-duplicated model changes the " ++ "computation graph without adding new parameters, trading extra compute for a " ++ "possible quality gain. Duplicating middle layers tends to help; duplicating " ++ "the first or last layers tends to break the model. The quick brown fox jumps " ++ "over the lazy dog while the patient scientist records each careful " ++ "measurement in a well-worn notebook by the light of a flickering lamp."; ++ ++int argmax_row(const float * row, int n_vocab) { ++ int best = 0; ++ float bv = row[0]; ++ for (int v = 1; v < n_vocab; ++v) { ++ if (row[v] > bv) { bv = row[v]; best = v; } ++ } ++ return best; ++} ++ ++// Compact perplexity over up to n_ctx tokens of `toks`, batched, accumulating ++// NLL incrementally so we never hold all logits at once. ++bool eval_ppl(llama_context * ctx, const llama_vocab * vocab, ++ const std::vector & toks, int n_ctx, int n_batch, ++ double & out_ppl) { ++ llama_memory_clear(llama_get_memory(ctx), true); ++ const int n_vocab = llama_vocab_n_tokens(vocab); ++ const int n = std::min((int) toks.size(), n_ctx); ++ if (n < 2) return false; ++ ++ llama_batch batch = llama_batch_init(n_batch, 0, 1); ++ double nll = 0.0; ++ long count = 0; ++ int pos = 0; ++ bool ok = true; ++ while (pos < n) { ++ const int cur = std::min(n_batch, n - pos); ++ batch.n_tokens = cur; ++ for (int k = 0; k < cur; ++k) { ++ batch.token[k] = toks[pos + k]; ++ batch.pos[k] = pos + k; ++ batch.n_seq_id[k] = 1; ++ batch.seq_id[k][0] = 0; ++ batch.logits[k] = 1; ++ } ++ if (llama_decode(ctx, batch)) { ok = false; break; } ++ const float * logits = llama_get_logits(ctx); ++ for (int k = 0; k < cur; ++k) { ++ const int gi = pos + k; ++ if (gi + 1 >= n) break; ++ const float * row = logits + (size_t) k * n_vocab; ++ float mx = row[0]; ++ for (int v = 1; v < n_vocab; ++v) if (row[v] > mx) mx = row[v]; ++ double se = 0.0; ++ for (int v = 0; v < n_vocab; ++v) se += std::exp((double) row[v] - mx); ++ const double lse = (double) mx + std::log(se); ++ const double lp = (double) row[toks[gi + 1]] - lse; ++ nll += -lp; ++ ++count; ++ } ++ pos += cur; ++ } ++ llama_batch_free(batch); ++ if (!ok || count == 0) return false; ++ out_ppl = std::exp(nll / (double) count); ++ return true; ++} ++ ++// Greedy generate a short answer per task; score = fraction whose answer string ++// appears in the generation. ++double eval_task(llama_context * ctx, const llama_vocab * vocab, int n_gen) { ++ const int n_vocab = llama_vocab_n_tokens(vocab); ++ const llama_token eos = llama_vocab_eos(vocab); ++ int correct = 0, total = 0; ++ llama_batch batch = llama_batch_init(512, 0, 1); ++ for (const TaskQA & t : kTasks) { ++ llama_memory_clear(llama_get_memory(ctx), true); ++ std::vector p = common_tokenize(vocab, t.q, true, false); ++ const int np = (int) p.size(); ++ if (np < 1 || np > 512) { ++total; continue; } ++ batch.n_tokens = np; ++ for (int k = 0; k < np; ++k) { ++ batch.token[k] = p[k]; ++ batch.pos[k] = k; ++ batch.n_seq_id[k] = 1; ++ batch.seq_id[k][0] = 0; ++ batch.logits[k] = (k == np - 1) ? 1 : 0; ++ } ++ if (llama_decode(ctx, batch)) { ++total; continue; } ++ std::string out; ++ int n_past = np; ++ bool hit = false; ++ llama_token tok = argmax_row(llama_get_logits_ith(ctx, -1), n_vocab); ++ for (int g = 0; g < n_gen; ++g) { ++ if (tok == eos) break; ++ out += common_token_to_piece(ctx, tok, false); ++ if (out.find(t.a) != std::string::npos) { hit = true; break; } // early-stop ++ batch.n_tokens = 1; ++ batch.token[0] = tok; ++ batch.pos[0] = n_past; ++ batch.n_seq_id[0] = 1; ++ batch.seq_id[0][0] = 0; ++ batch.logits[0] = 1; ++ if (llama_decode(ctx, batch)) break; ++ ++n_past; ++ tok = argmax_row(llama_get_logits_ith(ctx, -1), n_vocab); ++ } ++ ++total; ++ if (hit) ++correct; ++ } ++ llama_batch_free(batch); ++ return total ? (double) correct / (double) total : 0.0; ++} ++ ++// Build a context for this candidate plan, score it, free it. ++bool score_candidate(llama_model * model, const common_params & base, ++ const std::vector & corpus_toks, int n_gen, ++ Candidate & c) { ++ common_params p = base; // copy; `spec` must outlive the ctor call ++ const std::string spec = c.spec; ++ llama_context_params cp = common_context_params_to_llama(p); ++ cp.repeat_layers = spec.empty() ? nullptr : spec.c_str(); ++ llama_context * ctx = llama_init_from_model(model, cp); ++ if (!ctx) return false; // arch-unsupported / boundary reject / OOM ++ const llama_vocab * vocab = llama_model_get_vocab(model); ++ const bool ppl_ok = eval_ppl(ctx, vocab, corpus_toks, p.n_ctx, p.n_batch, c.ppl); ++ c.task = eval_task(ctx, vocab, n_gen); ++ llama_free(ctx); ++ c.ok = ppl_ok; ++ return ppl_ok; ++} ++ ++std::vector parse_int_list(const std::string & s) { ++ std::vector out; ++ size_t p = 0; ++ while (p < s.size()) { ++ size_t q = s.find(',', p); ++ if (q == std::string::npos) q = s.size(); ++ const std::string tok = s.substr(p, q - p); ++ if (!tok.empty()) { const int v = std::atoi(tok.c_str()); if (v > 0) out.push_back(v); } ++ p = q + 1; ++ } ++ return out; ++} ++ ++} // namespace ++ ++int rys_probe_main(int argc, char ** argv) { ++ // Unbuffered so our progress/table survives a GGML_ABORT (abort() skips the ++ // stdio flush; redirected streams are fully buffered → output would be lost). ++ setvbuf(stdout, nullptr, _IONBF, 0); ++ setvbuf(stderr, nullptr, _IONBF, 0); ++ ++ common_params params; ++ if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PERPLEXITY)) { ++ fprintf(stderr, "rys-probe: failed to parse arguments\n"); ++ return 1; ++ } ++ // The probe drives its own plans; ignore any user --repeat-layers. ++ params.repeat_layers.clear(); ++ // Disable auto memory-fit: the probe sizes its own small n_ctx per candidate, ++ // and the fit path builds a trial context (llama-context.cpp:469) whose ++ // flash_attn print aborted here. We don't want fit to mutate our cparams. ++ params.fit_params = false; ++ ++ llama_backend_init(); ++ llama_numa_init(params.numa); ++ ++ // Load the model ONCE (weights are plan-independent). model_only = no ctx. ++ common_init_result_ptr res = common_init_from_params(params, true); ++ llama_model * model = res ? res->model() : nullptr; ++ if (!model) { ++ fprintf(stderr, "rys-probe: failed to load model\n"); ++ return 1; ++ } ++ const llama_vocab * vocab = llama_model_get_vocab(model); ++ const int n_layer = llama_model_n_layer(model); ++ if (n_layer < 8) { ++ fprintf(stderr, "rys-probe: model too shallow (n_layer=%d)\n", n_layer); ++ return 1; ++ } ++ ++ // Corpus: user -f/-p prompt, else the small built-in. ++ const std::string corpus = params.prompt.empty() ? std::string(kDefaultCorpus) : params.prompt; ++ std::vector corpus_toks = common_tokenize(vocab, corpus, true, false); ++ ++ // Candidate band: default = the boundary-advisory mid-stack band. ++ const int band = std::max(3, n_layer / 8); ++ int lo = band, hi = n_layer - band; ++ if (params.rys_probe_band != "auto" && !params.rys_probe_band.empty()) { ++ const size_t colon = params.rys_probe_band.find(':'); ++ if (colon != std::string::npos) { ++ lo = std::atoi(params.rys_probe_band.substr(0, colon).c_str()); ++ hi = std::atoi(params.rys_probe_band.substr(colon + 1).c_str()); ++ } ++ } ++ lo = std::max(0, lo); ++ hi = std::min(n_layer, hi); ++ ++ // Sweep contiguous mid-stack blocks across widths. "auto" (default) = every ++ // width in the band, so we can find both a cheap-but-effective template and a ++ // max-gain one; else honor an explicit --rys-probe-widths list. ++ std::vector widths; ++ if (params.rys_probe_widths == "auto" || params.rys_probe_widths.empty()) { ++ for (int w = 1; w <= hi - lo; ++w) widths.push_back(w); ++ } else { ++ widths = parse_int_list(params.rys_probe_widths); ++ } ++ if (widths.empty()) widths = {1, 2, 4}; ++ ++ // Enumerate contiguous mid-stack blocks; cap the count with a stride. ++ std::vector cands; ++ for (int w : widths) { ++ for (int i = lo; i + w <= hi; ++i) { ++ Candidate c; ++ c.i = i; c.j = i + w; ++ c.spec = std::to_string(i) + "," + std::to_string(i + w); ++ c.overhead_pct = 100.0 * (double) w / (double) n_layer; ++ cands.push_back(c); ++ } ++ } ++ const size_t cap = 400; ++ if (cands.size() > cap) { ++ std::vector keep; ++ const double stride = (double) cands.size() / (double) cap; ++ for (double x = 0; (size_t) x < cands.size(); x += stride) keep.push_back(cands[(size_t) x]); ++ cands.swap(keep); ++ } ++ ++ fprintf(stderr, ++ "rys-probe: n_layer=%d band=[%d,%d) widths=%s candidates=%zu corpus_tokens=%zu\n", ++ n_layer, lo, hi, params.rys_probe_widths.c_str(), cands.size(), corpus_toks.size()); ++ ++ // Base (empty plan) reference first. ++ Candidate base; base.spec = ""; base.overhead_pct = 0.0; ++ if (!score_candidate(model, params, corpus_toks, 24, base)) { ++ fprintf(stderr, "rys-probe: base PPL failed — cannot score (bad corpus or model?)\n"); ++ return 1; ++ } ++ fprintf(stderr, "rys-probe: base ppl=%.4f task=%.3f\n", base.ppl, base.task); ++ ++ // Score each candidate. ++ size_t done = 0; ++ for (Candidate & c : cands) { ++ score_candidate(model, params, corpus_toks, 24, c); ++ c.dppl = c.ok ? (c.ppl - base.ppl) : 0.0; ++ ++done; ++ fprintf(stderr, "\rrys-probe: scored %zu/%zu (last %s: %s) ", ++ done, cands.size(), c.spec.c_str(), c.ok ? "ok" : "FAIL"); ++ fflush(stderr); ++ } ++ fprintf(stderr, "\n"); ++ ++ // Valid = beats base PPL AND does not regress the task-probe (guardrail so a ++ // boundary/broken block that happens to lower PPL can't win). ++ std::vector wins; ++ for (Candidate & c : cands) ++ if (c.ok && c.dppl < 0.0 && c.task >= base.task) wins.push_back(&c); ++ ++ // Context table: top-N wins by PPL gain (sorted ascending PPL). ++ std::sort(wins.begin(), wins.end(), ++ [](const Candidate * a, const Candidate * b) { return a->ppl < b->ppl; }); ++ const int topk = params.rys_probe_topk > 0 ? params.rys_probe_topk : 10; ++ printf("\n=== RYS probe (base ppl=%.4f, task=%.3f) — top wins by PPL ===\n", base.ppl, base.task); ++ printf("%-12s %5s %8s %10s %10s %8s\n", "block", "+lyr", "over%", "ppl", "dppl", "task"); ++ for (size_t k = 0; k < wins.size() && (int) k < topk; ++k) { ++ const Candidate * c = wins[k]; ++ printf("%-12s %5d %7.2f%% %10.4f %+10.4f %8.3f\n", ++ c->spec.c_str(), c->j - c->i, c->overhead_pct, c->ppl, c->dppl, c->task); ++ } ++ ++ if (wins.empty()) { ++ printf("\nNo candidate beat base PPL without regressing the task-probe. " ++ "Try a larger/domain corpus (-f) or a different --rys-probe-band.\n"); ++ printf("\nNOTE: PPL+task shortlist, not a verdict — validate on your real harness " ++ "(perf/llamafile/rys-probe-eval.sh; see docs/features/rys_probe.md).\n"); ++ return 0; ++ } ++ ++ // The two templates the user picks between: ++ // MAX GAIN = lowest PPL (task already >= base). ++ // MOST EFFICIENT = largest PPL drop PER ADDED LAYER (best bang per compute). ++ Candidate * gain = wins.front(); // sorted ascending by ppl ++ auto eff_of = [&](const Candidate * c) { ++ return (base.ppl - c->ppl) / (double) (c->j - c->i); ++ }; ++ Candidate * eff = wins.front(); ++ for (Candidate * c : wins) if (eff_of(c) > eff_of(eff)) eff = c; ++ ++ // Ready-to-paste bullet lines. "params" = extra layer weight re-run (RYS is ++ // weight-shared, so this is compute overhead, not extra param VRAM). ++ const int eff_n = eff->j - eff->i; ++ const int gain_n = gain->j - gain->i; ++ const double eff_p = (double) ((long) (eff->overhead_pct * 10 + 0.5)) / 10.0; // 1-dp, trimmed ++ const double gain_p = (double) ((long) (gain->overhead_pct * 10 + 0.5)) / 10.0; ++ ++ printf("\n=== 2 templates ===\n"); ++ printf("- MOST EFFICIENT → --repeat-layers %s — +%d %s, %g%% params, ppl %.2f→%.2f, task %.3f.\n", ++ eff->spec.c_str(), eff_n, eff_n == 1 ? "layer" : "layers", eff_p, ++ base.ppl, eff->ppl, eff->task); ++ printf("- MAX GAIN → --repeat-layers %s — +%d %s, %g%% params, ppl %.2f→%.2f, task %.3f.\n", ++ gain->spec.c_str(), gain_n, gain_n == 1 ? "layer" : "layers", gain_p, ++ base.ppl, gain->ppl, gain->task); ++ if (eff == gain) ++ printf(" (the same block is both — no wider block gained more.)\n"); ++ printf(" (detail: MOST EFFICIENT %.4f ppl/layer, dppl %+.4f; MAX GAIN dppl %+.4f. base ppl %.4f task %.3f.)\n", ++ eff_of(eff), eff->dppl, gain->dppl, base.ppl, base.task); ++ ++ printf("\nNOTE: PPL+task shortlist, not a verdict. Validate BOTH templates on a real task " ++ "eval with perf/llamafile/rys-probe-eval.sh (boots the server per spec + base and " ++ "runs your external harness). See docs/features/rys_probe.md.\n"); ++ ++ return 0; ++} ++ ++} // namespace lf diff --git a/patches/0124-rys-mtp-ctx-guard-bug2171.patch b/patches/0124-rys-mtp-ctx-guard-bug2171.patch new file mode 100644 index 0000000000000000000000000000000000000000..ac86422155c41f6b42e8aea802568a465c896583 --- /dev/null +++ b/patches/0124-rys-mtp-ctx-guard-bug2171.patch @@ -0,0 +1,23 @@ +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index df793cf7d..20423d5a0 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -244,9 +244,17 @@ llama_context::llama_context( + // (fail-loud is correct: a silently-ignored duplication request would mislead the operator). + // See docs/features/rys_layer_duplication.md. + { ++ // opencoti RYS×MTP (bug-2171): an MTP/NextN draft context NEVER applies the target's ++ // --repeat-layers duplication. RYS is a target-only inference capability; in self-spec the ++ // NextN head consumes the target's (already-RYS) hidden state via shared memory, so the draft ++ // runs the base layer stack. Applying the plan on the draft shadows the NextN head (canonical ++ // il=n_transformer) under an RYS-duplicated backbone slot -> "map_layer_ids MISS il=n_transformer" ++ // at draft-context init. The target context keeps RYS; the draft stays plain -> boots + lossless ++ // (target verifies every drafted token). See docs/features/rys_layer_duplication.md. ++ const char * rys_spec = (cparams.ctx_type == LLAMA_CONTEXT_TYPE_MTP) ? nullptr : params.repeat_layers; + const uint32_t n_transformer = hparams.n_layer - hparams.nextn_predict_layers; + std::string rys_err; +- if (!opencoti_parse_repeat_layers(params.repeat_layers, n_transformer, cparams.layer_plan, rys_err)) { ++ if (!opencoti_parse_repeat_layers(rys_spec, n_transformer, cparams.layer_plan, rys_err)) { + throw std::runtime_error("opencoti RYS: " + rys_err); + } + if (!cparams.layer_plan.empty()) { diff --git a/patches/0125-hybrid-window-orderagnostic-bug2172.patch b/patches/0125-hybrid-window-orderagnostic-bug2172.patch new file mode 100644 index 0000000000000000000000000000000000000000..710c66ee7d8a09f7d4b40dca1253d38cbd0f6116 --- /dev/null +++ b/patches/0125-hybrid-window-orderagnostic-bug2172.patch @@ -0,0 +1,267 @@ +opencoti patch 0125 — hybrid order-agnostic rolling-KV position-window (bug-2172 / #672): a hybrid (qwen35moe/qwen3next) attention sub-cache is fed by the recurrent batch pipeline (split_seq + recurrent rollback + find_slot cell-reuse/wrap) which produces NON-monotone destination cell idxs, tripping the append-only window scatter monotone assert (cpy_k) when rolling-KV spills. Fix (host-only, one translation unit): mem_attn is marked !append_only; in window mode its idx tensors become 2-row [n_tokens,2] routing tensors (row 0 window, row 1 tail), window/tail tensors get +1 trash sink cell, and cpy_k/cpy_v scatter the full per-stream K/V into BOTH window and tail via arbitrary-dest ggml_set_rows — off-target copies land in the never-read trash cell. Read side is unchanged (full window+tail regions, FA mask zeros inactive cells — mask contract verified). Every existing (append-only) cache keeps the byte-identical fast path. +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +index 497e9dcc1..7045ec55e 100644 +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -513,6 +513,26 @@ private: + + bool v_trans = true; // the value tensor is transposed + ++ // opencoti bug-2172 (#672): position-window scatter/read fast path (contiguous ++ // window prefix [0,n_win) + tail suffix) assumes APPEND-ONLY placement — dest ++ // cell idx == key position, monotone in batch order. Every normal cache ++ // (dense/iSWA/unified) satisfies this, so append_only stays true and the hot ++ // paths are byte-identical + unchanged. The HYBRID wrapper's attention ++ // sub-cache is driven by the recurrent batch pipeline (split_seq + recurrent ++ // rollback + find_slot cell-reuse/wrap) → non-monotone dest idxs → the ++ // fast-path monotone GGML_ASSERT (cpy_k) would fire. When false (set ONLY by ++ // llama_memory_hybrid for mem_attn), cpy_k/cpy_v take an order-agnostic ++ // gathered scatter (fixed 2 get_rows + 2 set_rows topology, no graph ++ // re-capture) and get_k/get_v read the FULL window/tail regions with the FA ++ // mask zeroing inactive cells. Cost is paid ONLY by hybrids under spill — ++ // exactly the case that is otherwise broken — never by any existing model. ++ // opencoti-hook: hybrid-window-orderagnostic — bug-2172 (#672), see docs/features/rolling_kv.md ++ bool append_only = true; ++public: ++ // set ONLY by llama_memory_hybrid on its attention sub-cache (needs external access) ++ void set_append_only(bool v) { append_only = v; } ++private: ++ + const uint32_t n_seq_max = 1; + const uint32_t n_stream = 1; + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 932500436..7057531d6 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -1103,6 +1103,18 @@ llama_kv_cache::llama_kv_cache( + const bool layer_window = window_mode && !layer_local; + const uint32_t wc = layer_window ? window_cells : kv_size; + const uint32_t tail_c = layer_window ? (kv_size - window_cells) : 0; ++ // opencoti bug-2172 (#672): in window mode allocate ONE extra "trash" cell in ++ // both the resident window (k/v_per_stream) and the host tail ++ // (k/v_cpu_per_stream). The order-agnostic scatter used by a hybrid ++ // (non-append-only) mem_attn cache — whose recurrent rollback produces ++ // non-monotone destination idxs — routes each new token to BOTH the window ++ // and the tail tensor, sending the off-target copy to this never-read sink ++ // (see cpy_k / set_input_k_idxs). The append-only fast path never touches the ++ // trash cell (it writes/reads only [0,wc) / [0,tail_c)), so its compute is ++ // byte-identical; the cost is 1 cell/layer/stream (negligible VRAM). Outside ++ // window mode (layer_window==false) nothing changes. ++ const uint32_t wc_alloc = wc + (layer_window ? 1u : 0u); ++ const uint32_t tail_c_alloc = tail_c + (layer_window ? 1u : 0u); + + // opencoti F4 M3 Phase 4 — per-stream tensor split. + // Each stream gets its own OWNING 3-D tensor of shape +@@ -1134,10 +1146,10 @@ llama_kv_cache::llama_kv_cache( + // opencoti F5 M7 Stage 3c — `wc` is kv_size outside window mode + // (byte-identical), or the resident window length in window mode. + ggml_tensor * k_s = has_k +- ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, wc, 1) ++ ? ggml_new_tensor_3d(ctx_s, type_k, n_embd_k_gpu, wc_alloc, 1) // opencoti bug-2172: +1 trash cell in window mode + : nullptr; + ggml_tensor * v_s = has_v +- ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, wc, 1) ++ ? ggml_new_tensor_3d(ctx_s, type_v, n_embd_v_gpu, wc_alloc, 1) // opencoti bug-2172: +1 trash cell in window mode + : nullptr; + + // Tensor naming preserves the pre-Phase-4 name in unified +@@ -1196,7 +1208,7 @@ llama_kv_cache::llama_kv_cache( + // window mode: ALL heads, tail_c cells; M2: CPU head subset, kv_size cells. + const uint32_t host_k_embd = layer_window ? n_embd_k_gqa : n_embd_k_cpu; + const uint32_t host_v_embd = layer_window ? n_embd_v_gqa : n_embd_v_cpu; +- const uint32_t host_cells = layer_window ? tail_c : kv_size; ++ const uint32_t host_cells = layer_window ? tail_c_alloc : kv_size; // opencoti bug-2172: +1 trash cell in window mode + // opencoti #582-P2 (mixed-type KV): the position-tail may store a + // MORE-compressed type than the resident window. `type_k_tail` / + // `type_v_tail` default to GGML_TYPE_COUNT (== "same as window"), +@@ -4053,6 +4065,41 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); + GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); + ++ // opencoti-hook: hybrid-window-orderagnostic — bug-2172 (#672). ++ // opencoti bug-2172 (#672): order-agnostic scatter for a hybrid (non-append-only) ++ // mem_attn cache. Recurrent rollback produces non-monotone destination idxs, so ++ // window rows (idx=wc) are interleaved and the append-only ++ // "window prefix / tail suffix" partition below is invalid. k_idxs is [n_tokens, 2] ++ // (build_input_k_idxs): row 0 = window-routing, row 1 = tail-routing, off-target ++ // tokens routed to the +1 trash sink. Scatter the FULL per-stream k_cur2 into BOTH ++ // k_s and kc_s — ggml_set_rows honors arbitrary (non-sorted) destination idxs, and ++ // the trash-bound copies land in the never-read sink cell. Same op count at reserve ++ // and live (exactly 2 set_rows per stream), so the reserve topology matches. ++ if (!append_only) { ++ GGML_ASSERT(k_idxs->ne[1] == 2 && "hybrid windowed cpy_k needs the 2-row k_idxs routing tensor"); ++ ggml_tensor * tie2 = nullptr; ++ auto acc2 = [&](ggml_tensor * store) { ++ if (!tie2) { tie2 = store; return; } ++ tie2 = ggml_add(ctx, ++ ggml_cast(ctx, ggml_view_1d(ctx, tie2, ggml_blck_size(tie2->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ }; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[s]]; ++ ggml_tensor * kc_s = layer.k_cpu_per_stream[sinfo.strm[s]]; ++ const size_t base = (size_t) s * tokens_per_stream; ++ ggml_tensor * cur = ggml_view_2d(ctx, k_cur2, n_embd_gqa, tokens_per_stream, ++ k_cur2->nb[1], base * k_cur2->nb[1]); ++ ggml_tensor * iw = ggml_view_1d(ctx, k_idxs, tokens_per_stream, ++ base * k_idxs->nb[0]); // row 0 (window) ++ ggml_tensor * it = ggml_view_1d(ctx, k_idxs, tokens_per_stream, ++ ((size_t) n_tokens + base) * k_idxs->nb[0]); // row 1 (tail) ++ acc2(ggml_set_rows(ctx, k_s, cur, iw)); ++ acc2(ggml_set_rows(ctx, kc_s, cur, it)); ++ } ++ return tie2; ++ } ++ + ggml_tensor * tie = nullptr; + auto accumulate = [&](ggml_tensor * store) { + if (!tie) { tie = store; return; } +@@ -4231,6 +4278,35 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + const int64_t tokens_per_stream = (batch_ns == 1) ? n_tokens : (n_tokens / (int64_t) batch_ns); + GGML_ASSERT((int64_t) batch_ns * tokens_per_stream == n_tokens); + ++ // opencoti-hook: hybrid-window-orderagnostic — bug-2172 (#672). ++ // opencoti bug-2172 (#672): order-agnostic scatter for a hybrid (non-append-only) ++ // mem_attn cache (mirror of cpy_k). v_idxs is [n_tokens, 2] (build_input_v_idxs): ++ // row 0 = window-routing, row 1 = tail-routing (off-target → +1 trash sink). ++ if (!append_only) { ++ GGML_ASSERT(v_idxs->ne[1] == 2 && "hybrid windowed cpy_v needs the 2-row v_idxs routing tensor"); ++ ggml_tensor * tie2 = nullptr; ++ auto acc2 = [&](ggml_tensor * store) { ++ if (!tie2) { tie2 = store; return; } ++ tie2 = ggml_add(ctx, ++ ggml_cast(ctx, ggml_view_1d(ctx, tie2, ggml_blck_size(tie2->type), 0), GGML_TYPE_F32), ++ ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ }; ++ for (uint32_t s = 0; s < batch_ns; ++s) { ++ ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[s]]; ++ ggml_tensor * vc_s = layer.v_cpu_per_stream[sinfo.strm[s]]; ++ const size_t base = (size_t) s * tokens_per_stream; ++ ggml_tensor * cur = ggml_view_2d(ctx, v_cur2, n_embd_gqa, tokens_per_stream, ++ v_cur2->nb[1], base * v_cur2->nb[1]); ++ ggml_tensor * iw = ggml_view_1d(ctx, v_idxs, tokens_per_stream, ++ base * v_idxs->nb[0]); ++ ggml_tensor * it = ggml_view_1d(ctx, v_idxs, tokens_per_stream, ++ ((size_t) n_tokens + base) * v_idxs->nb[0]); ++ acc2(ggml_set_rows(ctx, v_s, cur, iw)); ++ acc2(ggml_set_rows(ctx, vc_s, cur, it)); ++ } ++ return tie2; ++ } ++ + ggml_tensor * tie = nullptr; + auto accumulate = [&](ggml_tensor * store) { + if (!tie) { tie = store; return; } +@@ -4323,7 +4399,17 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + ggml_tensor * llama_kv_cache::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const { + const uint32_t n_tokens = ubatch.n_tokens; + +- ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); ++ // opencoti bug-2172 (#672): a hybrid (non-append-only) windowed mem_attn cache scatters ++ // each new token to BOTH the resident window and the host tail (its recurrent rollback ++ // produces non-monotone dest idxs, so the append-only "window prefix / tail suffix" ++ // partition is invalid). Carry the two routings as a 2-row index tensor: row 0 = ++ // window-routing (dest cell or the window trash sink), row 1 = tail-routing (dest cell ++ // or the tail trash sink). ne[0] stays n_tokens so every `k_idxs->ne[0] == n_tokens` ++ // graph guard is unaffected. append-only caches keep the byte-identical 1-D tensor. ++ const bool windowed = !layers.empty() && layers.front().window_cells > 0; ++ ggml_tensor * k_idxs = (!append_only && windowed) ++ ? ggml_new_tensor_2d(ctx, GGML_TYPE_I64, n_tokens, 2) ++ : ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); + + ggml_set_input(k_idxs); + +@@ -4336,7 +4422,12 @@ ggml_tensor * llama_kv_cache::build_input_v_idxs(ggml_context * ctx, const llama + ggml_tensor * v_idxs; + + if (!v_trans) { +- v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); ++ // opencoti bug-2172 (#672): 2-row routing for the hybrid windowed path (mirror of ++ // build_input_k_idxs). Window mode is gated to !v_trans, so this only applies here. ++ const bool windowed = !layers.empty() && layers.front().window_cells > 0; ++ v_idxs = (!append_only && windowed) ++ ? ggml_new_tensor_2d(ctx, GGML_TYPE_I64, n_tokens, 2) ++ : ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); + } else { + v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens*hparams.n_embd_v_gqa_max()); + } +@@ -4413,6 +4504,29 @@ void llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ub + // idx-wc; idx < wc stays window-local. wc is uniform across the cache's layers. + // wc == 0 (no window) ⇒ byte-identical to the vanilla local indexing below. + const uint32_t wc_k = layers.empty() ? 0u : layers.front().window_cells; ++ ++ // opencoti bug-2172 (#672): 2-row routing layout for a hybrid (non-append-only) windowed ++ // cache (dst is [n_tokens, 2] — see build_input_k_idxs). Recurrent rollback makes the ++ // dest idxs non-monotone, so cpy_k scatters EVERY token to both the resident window and ++ // the host tail. Fill row 0 = window-routing (window-local dest, or the window trash sink ++ // for tail-bound tokens) and row 1 = tail-routing (tail-local dest, or the tail trash ++ // sink for window-bound tokens). The trash cells are the +1 slot allocated past ++ // [0,wc) / [0,tail_c) (derive from the tensor sizes so this can't drift). ++ if (dst->ne[1] == 2 && wc_k > 0 && !layers.front().k_per_stream.empty() ++ && !layers.front().k_cpu_per_stream.empty() && layers.front().k_cpu_per_stream[0]) { ++ const int64_t win_trash = layers.front().k_per_stream[0]->ne[1] - 1; // == wc ++ const int64_t tail_trash = layers.front().k_cpu_per_stream[0]->ne[1] - 1; // == tail_c ++ for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { ++ for (uint32_t i = 0; i < sinfo.size(); ++i) { ++ const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; ++ const int64_t slot = (int64_t) s*sinfo.size() + i; ++ data[slot] = (idx < wc_k) ? (int64_t) idx : win_trash; // row 0 (window) ++ data[n_tokens + slot] = (idx >= wc_k) ? (int64_t) (idx - wc_k) : tail_trash; // row 1 (tail) ++ } ++ } ++ return; ++ } ++ + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + for (uint32_t i = 0; i < sinfo.size(); ++i) { + const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; +@@ -4438,6 +4552,25 @@ void llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ub + // set_input_k_idxs). window mode is gated to !v_trans, so the tail-local + // adjustment only applies here. wc == 0 ⇒ byte-identical to vanilla. + const uint32_t wc_v = layers.empty() ? 0u : layers.front().window_cells; ++ ++ // opencoti bug-2172 (#672): 2-row routing for a hybrid (non-append-only) windowed ++ // cache (mirror of set_input_k_idxs). row 0 = window-routing, row 1 = tail-routing; ++ // off-target tokens → the +1 trash sink derived from the tensor sizes. ++ if (dst->ne[1] == 2 && wc_v > 0 && !layers.front().v_per_stream.empty() ++ && !layers.front().v_cpu_per_stream.empty() && layers.front().v_cpu_per_stream[0]) { ++ const int64_t win_trash = layers.front().v_per_stream[0]->ne[1] - 1; // == wc ++ const int64_t tail_trash = layers.front().v_cpu_per_stream[0]->ne[1] - 1; // == tail_c ++ for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { ++ for (uint32_t i = 0; i < sinfo.size(); ++i) { ++ const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; ++ const int64_t slot = (int64_t) s*sinfo.size() + i; ++ data[slot] = (idx < wc_v) ? (int64_t) idx : win_trash; ++ data[n_tokens + slot] = (idx >= wc_v) ? (int64_t) (idx - wc_v) : tail_trash; ++ } ++ } ++ return; ++ } ++ + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + for (uint32_t i = 0; i < sinfo.size(); ++i) { + const uint32_t idx = (uint32_t) sinfo.idxs[s][i]; +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +index fa336c2c5..67ecee16c 100644 +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -101,7 +101,17 @@ llama_memory_hybrid::llama_memory_hybrid( + // opencoti RYS (#666) — forward the layer plan so a duplicated recurrent + // (gated-delta-net) layer gets its own state slot (eff-keyed). + layer_plan +- )) {} ++ )) { ++ // opencoti bug-2172 (#672): the attention sub-cache of a hybrid is fed by the ++ // recurrent batch pipeline (split_seq + [TAG_RECURRENT_ROLLBACK_SPLITS] + ++ // find_slot cell-reuse/wrap), which produces NON-monotone destination cell ++ // idxs. Mark it non-append-only so the position-window scatter/read take the ++ // order-agnostic paths (gathered set_rows + full-region masked FA read) ++ // instead of the append-only contiguous fast path, whose monotone assert ++ // would otherwise fire when the rolling-KV window spills. All other caches ++ // stay append_only=true (byte-identical fast path). ++ mem_attn->set_append_only(false); ++} + + llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { + do { diff --git a/patches/0126-p0-dcaoff-scalar-mma-decode.patch b/patches/0126-p0-dcaoff-scalar-mma-decode.patch new file mode 100644 index 0000000000000000000000000000000000000000..021ae40e4c6a99d8708698d34690e964c7a65946 --- /dev/null +++ b/patches/0126-p0-dcaoff-scalar-mma-decode.patch @@ -0,0 +1,1024 @@ +opencoti patch 0126 — P0 DCA-off scalar-quant MMA decode (#582-P0 / #620, #644-mirror) + +Single-token quant-KV decode with DCA OFF was stuck on the FA-VEC reader, which +HAL §3b measured as kernel-bound (~2x f16, regardless of bytes moved). f16 at +high-GQA / long-KV already falls to the tensor-core MMA path (the fattn.cu +selector); this patch gives a scalar-quant K/V cache the SAME route. It reads the +raw quant blocks straight into the MMA smem tile in-register +(load_tile_dequant / get_dequantize_V — no whole-KV materialize) and runs the +tensor-core mul_mat. Quant decode therefore becomes bandwidth-bound and BEATS f16 +(fewer KV bytes moved). This is the DCA-off mirror of 0119's DCA-on scalar +in-register read. + +opencoti's CUDA build passes --fa-all-quants (build-pipeline.ts), so a native +FA-VEC instance for every scalar pair already EXISTS — this is an MMA fast-path +OVER that VEC path, NOT new support. Routed by the CUDA selector only when +OPENCOTI_QUANT_MMA_DECODE != 0 (default-on); env=0 reverts each pair to its +native FA-VEC path (graceful, no refusal). Scalar KV is not FWHT-rotated, so +there is no wht_o / de-rotate (unlike the turbo verify kernel #630). + +MATRIX (D128 served pairs, mirrors the #644 scalar set): hi-K/cheap-V +{q8K/{q4,q5,q6,q8}V, q6K/q4V} + symmetric diagonal {q4_0,q4_1,q5_0,q5_1,q6_0}. + +RESULTS (bs2 GPU1, Qwen2.5-14B-1M Q8_0, ctx 65536, n_kv~44k): + beachhead q8_0-K/q4_0-V : 70.45 tps vs f16 60.99 (1.15x) vs native VEC 30.60 + (2.30x — the §3b kernel-bound tax fully killed); real_frac 0.0 vs BOTH f16 + and native VEC (top-token identical — lossless). + generalization (5-config, ctx 65536): decode tps rises monotonically as V gets + cheaper — f16 62.15 < q8q4 68.47 < q8q8 69.45 < q6q4 70.31 < q4q4 73.97; all + boot coherent, real_frac 0.0 vs f16. + +MECHANISM (all additive; opencoti-hook: P0 DCA-off scalar-quant MMA decode): + - fattn.cu: 10 forward-decls + octi_quant_mma_decode_enabled() (env gate, + default-on) + shared predicate octi_scalar_mma_pair_d128(K,V) (the served + D128 scalar pair set); MMA-dispatch block (OCTI_SCALAR_MMA_D128 macro over the + 10 pairs) in ggml_cuda_flash_attn_ext_mma_f16; guard admittance + + VEC-selector escape (both gated on dst->src[7]==nullptr = DCA off) in + ggml_cuda_get_best_fattn_kernel so the pair falls straight to MMA_F16. + - 10 new template-instances/fattn-mma-f16-scalar---d128.cu: each a typed + entry + ncols1/ncols2 gqa dispatch calling + ggml_cuda_flash_attn_ext_mma_f16_case<128,128,...,GGML_TYPE_,GGML_TYPE_>. + Auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531). + +Correctness gated by logit-equiv (real_frac), never greedy needle (bug-263/270). +CUDA-only change; host binary unchanged. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -141,6 +141,54 @@ + void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620, generalized to the #644 scalar-D128 pair set) — ++// plain (non-DCA) scalar-quant MMA DECODE entries (template-instances/fattn-mma-f16-scalar---d128.cu). ++// Each reads a raw quant K/V cache into the MMA smem tile in-register (load_tile_dequant / get_dequantize_V, ++// no whole-KV materialize), giving quant decode the tensor-core path f16 already gets at high-GQA/long-KV. ++// NOTE: opencoti's CUDA build passes --fa-all-quants (build-pipeline.ts), so every pair below ALSO has a ++// native FA-VEC instance; env=0 keeps that VEC route (kernel-bound ~2x f16 per HAL §3b), env=1 (default) ++// takes this MMA fast-path. Validated q8K/q4V: 70.45 vs f16 60.99 tps (1.15x), 2.30x over native VEC, ++// real_frac=0 (Qwen2.5-14B-1M, 65k). Default/beachhead pair = q8K/q4V (the P1 auto-tier default). ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++ ++// OPENCOTI_QUANT_MMA_DECODE — DEFAULT-ON; =0 disables the MMA fast-path (each pair reverts to its native ++// FA-VEC route). Gates both the decode selector escape (below) and — only for a hypothetical non-all-quants ++// build — the K!=V supports_op admittance of the asymmetric pairs (that admittance is dead code under ++// opencoti's --fa-all-quants build, kept for vanilla-tree parity). Byte-identical no-op for every non-listed ++// config. ++static bool octi_quant_mma_decode_enabled() { ++ static const bool en = []() { const char * e = getenv("OPENCOTI_QUANT_MMA_DECODE"); return (!e || e[0] != '0'); }(); ++ return en; ++} ++ ++// The served DCA-off scalar-MMA decode pairs (mirror of the #644 DCA-on scalar set), D128 only. Both the K!=V ++// guard admittance, the decode selector escape, and the MMA dispatch consult this single predicate so they can ++// never disagree on which pairs are diverted to the tensor-core instances. ++static bool octi_scalar_mma_pair_d128(const ggml_tensor * K, const ggml_tensor * V) { ++ if (K->ne[0] != 128 || V->ne[0] != 128) { ++ return false; ++ } ++ const ggml_type vt = V->type; ++ switch (K->type) { ++ case GGML_TYPE_Q8_0: return vt == GGML_TYPE_Q8_0 || vt == GGML_TYPE_Q6_0 || vt == GGML_TYPE_Q5_0 || vt == GGML_TYPE_Q4_0; ++ case GGML_TYPE_Q6_0: return vt == GGML_TYPE_Q6_0 || vt == GGML_TYPE_Q4_0; ++ case GGML_TYPE_Q5_0: return vt == GGML_TYPE_Q5_0; ++ case GGML_TYPE_Q5_1: return vt == GGML_TYPE_Q5_1; ++ case GGML_TYPE_Q4_0: return vt == GGML_TYPE_Q4_0; ++ case GGML_TYPE_Q4_1: return vt == GGML_TYPE_Q4_1; ++ default: return false; ++ } ++} ++ + // opencoti-hook: WS1 turbo-MMA (#628/#630) — per-head-dim OPT-IN knobs (default OFF). D256 = Gemma SWA-local, + // D512 = Gemma global, D128 = Qwen full-attention. When off, the scalar-K/turbo-V verify batch keeps the + // fused-VEC routing (the pre-#628 default + logit-equiv reference). +@@ -174,6 +222,28 @@ + if (Q->ne[0] == 512) { ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ctx, dst); return; } + } + ++ // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — the selector routes a served D128 scalar-K/ ++ // scalar-V decode (see octi_scalar_mma_pair_d128) to MMA_F16; hand it to the typed instance (raw in-register ++ // quant read) instead of the f16 Q->ne[0] switch below (which would reinterpret the quant blocks as raw ++ // f16). Only the served pairs are diverted; all other quant K/V keep the default switch. Guarded by the ++ // same predicate the selector uses — only reached when that selector already returned MMA_F16 for the pair. ++ if (octi_quant_mma_decode_enabled() && octi_scalar_mma_pair_d128(K, V)) { ++ const ggml_type vt = V->type; ++#define OCTI_SCALAR_MMA_D128(kk, vv, fn) \ ++ if (K->type == GGML_TYPE_##kk && vt == GGML_TYPE_##vv) { fn(ctx, dst); return; } ++ OCTI_SCALAR_MMA_D128(Q8_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d128) ++ OCTI_SCALAR_MMA_D128(Q8_0, Q8_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d128) ++ OCTI_SCALAR_MMA_D128(Q8_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d128) ++ OCTI_SCALAR_MMA_D128(Q8_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d128) ++ OCTI_SCALAR_MMA_D128(Q6_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d128) ++ OCTI_SCALAR_MMA_D128(Q6_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d128) ++ OCTI_SCALAR_MMA_D128(Q5_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d128) ++ OCTI_SCALAR_MMA_D128(Q5_1, Q5_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d128) ++ OCTI_SCALAR_MMA_D128(Q4_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d128) ++ OCTI_SCALAR_MMA_D128(Q4_1, Q4_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d128) ++#undef OCTI_SCALAR_MMA_D128 ++ } ++ + switch (Q->ne[0]) { + case 64: + GGML_ASSERT(V->ne[0] == 64); +@@ -570,7 +640,18 @@ + K->type == GGML_TYPE_Q6_0 || K->type == GGML_TYPE_Q8_0) && + (V->type == GGML_TYPE_TURBO2_0 || V->type == GGML_TYPE_TURBO3_0 || + V->type == GGML_TYPE_TURBO2_TCQ || V->type == GGML_TYPE_TURBO3_TCQ); +- if (K->type != V->type && !both_tcq && !scalar_k_turbo_v) { ++ // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — admit the served asymmetric scalar pairs ++ // (octi_scalar_mma_pair_d128, e.g. q8_0-K/q4_0-V) so the selector below can route their decode to the ++ // scalar-MMA instance. This whole block is #ifndef GGML_CUDA_FA_ALL_QUANTS, so under opencoti's actual ++ // --fa-all-quants build it is COMPILED OUT (every pair already has a native FA-VEC instance and reaches ++ // the selector); this admittance is the fallback that keeps the feature working on a hypothetical ++ // non-all-quants / vanilla tree. Env-gated: off ⇒ pre-#620 status quo. dst->src[7] != nullptr ⇒ ++ // sparse-attn (vslash) block-selector active, which the selector forces onto the FA-VEC block_sel reader ++ // (no scalar-MMA block_sel path) — keep the pair UNSUPPORTED in that combo (graceful NONE) rather than ++ // admitting it into a dispatch with no matching kernel. ++ const bool scalar_k_scalar_v_mma = ++ octi_quant_mma_decode_enabled() && octi_scalar_mma_pair_d128(K, V) && dst->src[7] == nullptr; ++ if (K->type != V->type && !both_tcq && !scalar_k_turbo_v && !scalar_k_scalar_v_mma) { + return BEST_FATTN_KERNEL_NONE; + } + } +@@ -677,30 +758,42 @@ + // If Turing tensor cores are available, use them: + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { +- if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { +- // opencoti bug-858 (2026-07-05): REVERTED the earlier <=2 experiment back to upstream's +- // Q->ne[1]==1. Routing the n_q=2 verify to VEC (cols_per_block=2) did NOT make it bit-equal +- // to the n_q=1 decode (different column tiling → different accumulation order) and measurably +- // WORSENED dual-vs-plain losslessness at n_max=1 (gate4b: 0/4 prompts byte-equal vs 1/4 on +- // upstream ==1 semantics; the earlier accept win attributed to this change was actually the +- // output_all/masked-tap defect, fixed in common/speculative.cpp). Keep upstream semantics: +- // VEC for decode only; verify (n_q>=2) takes MMA exactly like b9859. +- if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) { +- return BEST_FATTN_KERNEL_VEC; +- } +- } else { +- if (cc >= GGML_CUDA_CC_ADA_LOVELACE) { +- if (Q->ne[1] <= 2) { ++ // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — for a served scalar pair @ D128 ++ // (octi_scalar_mma_pair_d128, default-on) skip the ENTIRE VEC-decision so it falls straight to MMA ++ // below, for every n_q and every n_kv (prefill, decode, short- or long-context). Unlike f16 (which ++ // keeps its VEC decode path and only escapes to MMA at high-GQA/long-KV), quant reclaim is ++ // unconditional for these pairs — the whole point is to take quant decode OFF the kernel-bound ++ // FA-VEC reader (HAL §3b) and onto the tensor cores. env=0 leaves the pair on its native FA-VEC ++ // route (opencoti builds --fa-all-quants, so that VEC instance exists — no crash, just slower). ++ // dst->src[7] guard: sparse-attn (vslash) keeps the FA-VEC block_sel reader (no scalar-MMA block_sel). ++ const bool octi_scalar_mma = octi_quant_mma_decode_enabled() ++ && octi_scalar_mma_pair_d128(K, V) && dst->src[7] == nullptr; ++ if (!octi_scalar_mma) { ++ if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { ++ // opencoti bug-858 (2026-07-05): REVERTED the earlier <=2 experiment back to upstream's ++ // Q->ne[1]==1. Routing the n_q=2 verify to VEC (cols_per_block=2) did NOT make it bit-equal ++ // to the n_q=1 decode (different column tiling → different accumulation order) and measurably ++ // WORSENED dual-vs-plain losslessness at n_max=1 (gate4b: 0/4 prompts byte-equal vs 1/4 on ++ // upstream ==1 semantics; the earlier accept win attributed to this change was actually the ++ // output_all/masked-tap defect, fixed in common/speculative.cpp). Keep upstream semantics: ++ // VEC for decode only; verify (n_q>=2) takes MMA exactly like b9859. ++ if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) { + return BEST_FATTN_KERNEL_VEC; + } + } else { +- if (Q->ne[1] == 1) { +- return BEST_FATTN_KERNEL_VEC; ++ if (cc >= GGML_CUDA_CC_ADA_LOVELACE) { ++ if (Q->ne[1] <= 2) { ++ return BEST_FATTN_KERNEL_VEC; ++ } ++ } else { ++ if (Q->ne[1] == 1) { ++ return BEST_FATTN_KERNEL_VEC; ++ } + } + } +- } +- if (!gqa_opt_applies && Q->ne[1] == 1) { +- return BEST_FATTN_KERNEL_VEC; ++ if (!gqa_opt_applies && Q->ne[1] == 1) { ++ return BEST_FATTN_KERNEL_VEC; ++ } + } + } + return BEST_FATTN_KERNEL_MMA_F16; +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_0_q4_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_1-K / Q4_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_1/q4_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_1_q4_1_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_0_q5_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_1-K / Q5_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_1/q5_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_1_q5_1_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q4_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q6_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q4_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q5_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q6_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d128_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d128.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d128.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d128.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620/#644-mirror) — D128 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q8_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q8_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==128 (full gqa 1/2/4/8), matching the turbo-MMA D128 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q8_0_d128_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D128 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d128_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d128_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d128_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d128_ncols1<1>(ctx, dst); ++} diff --git a/patches/0127-p0-dcaoff-scalar-mma-decode-d256-d512.patch b/patches/0127-p0-dcaoff-scalar-mma-decode-d256-d512.patch new file mode 100644 index 0000000000000000000000000000000000000000..3b552b1494d9d309203be08bb3526137ccdd91bb --- /dev/null +++ b/patches/0127-p0-dcaoff-scalar-mma-decode-d256-d512.patch @@ -0,0 +1,1830 @@ +opencoti patch 0127 — P0 DCA-off scalar-quant MMA decode: D256/D512 generalization (#582-P0 / #674) + +0126 (#620) gave a scalar-quant K/V cache the tensor-core MMA decode fast-path on D128 +(Qwen full-attention). This patch generalizes that fast-path to Gemma-4's two head-dims: +D256 (SWA-local layers) and D512 (global layers — the KV that grows with context, where +the HAL §3c/§3d −28..35% kernel-bound quant tax bit hardest). One boot now routes a Gemma +scalar-KV decode through the tensor cores on every layer via the per-layer head-dim +predicate — no whole-KV materialize, raw quant blocks read straight into the MMA smem tile +in-register (load_tile_dequant / get_dequantize_V). Quant decode becomes bandwidth-bound +and BEATS f16 (fewer KV bytes moved), which is what makes scalar-quant KV the preferred +high-VRAM serving path over turbo (turbo is slower AND lower quality). + +The mechanism is 0126's, widened: the shared predicate octi_scalar_mma_pair now admits +symmetric head-dim K==V ∈ {128,256,512} (was 128-only), and the MMA dispatch routes by +head-dim to the _d128 / _d256 / _d512 typed instance. The two consult-sites (the +--fa-all-quants dead-code guard admittance + the selector escape) already read only that +predicate, so they generalize for free. env OPENCOTI_QUANT_MMA_DECODE default-on; env=0 +reverts each pair to its native FA-VEC route (graceful, no refusal) at every head-dim. + +RESULTS (bs2 GPU1, Gemma-4 A4B native google 26B-A4B-128e Q4_K_M, DCA-off, logit-equiv +vs f16 — Gemma A4B PPL/KLD is garbage, so correctness is top-token identity): + ctx 64k (nkv 42.5k): f16 162.13 < q4q4 170.17 < q8q8 173.86 < q8q4 174.94 tps + (q8q4 +7.9% over f16; env=0 VEC fallback 122.17 = the §3c kernel-bound tax, so MMA is + +43% over VEC — tax fully erased). real_frac 0.0 for q8q4/q8q8/q4q4 vs f16 (lossless). + ctx 128k (nkv 112k): f16 152.15 < q4q4 152.95 < q8q4 154.46 tps (q8q4 +1.5%; §3c had + q8q4-VEC at −35% here). real_frac 0.0 for q8q4/q4q4 (lossless at depth). Win narrows + deep as MoE-FFN + per-step launch overhead dilute the KV-bandwidth delta, but scalar + stays ≥ f16 and top-token-lossless at both depths. + +MATRIX: the full #644 D128 scalar set at each of D256/D512 (10 pairs × 2 dims = 20 new TUs): + hi-K/cheap-V {q8K/{q4,q5,q6,q8}V, q6K/q4V} + symmetric diagonal {q4_0,q4_1,q5_0,q5_1,q6_0}. + +MECHANISM (all additive; opencoti-hook: #582-P0 DCA-off quant-MMA decode #620 D128 + #674 D256/D512): + - fattn.cu: forward-decls now an OCTI_DECL_SCALAR_MMA(kk,vv) macro emitting all three + head-dims; octi_scalar_mma_pair_d128 -> octi_scalar_mma_pair (admits K==V ∈ {128,256,512}); + the OCTI_SCALAR_MMA_D128 dispatch macro -> OCTI_SCALAR_MMA(kk,vv,base) routing on head-dim + d ∈ {128,256,512} to base##_d{128,256,512}; comments/consult-site names updated. + - 20 new template-instances/fattn-mma-f16-scalar---d{256,512}.cu (D256/D512 twins of + the 0126 D128 TUs; ggml_cuda_flash_attn_ext_mma_f16_case<256,256,...> / <512,512,...>). + D256/D512 MMA + in-register scalar V-readers (ne==8) are proven by the pre-existing turbo + D256/D512 instances (#630) and f16's switch_ncols2<256,256>/<512,512>. Auto-collected by + build-functions.sh's fattn-mma-*.cu glob (#531). + +Build note: the 20 new (esp. D512) instances grow ggml-cuda.so past 4 GiB (~4.43 GiB), which +crossed a latent build-pipeline bug (bug-2178): sha256OfFile read the whole DSO into one +ArrayBuffer and aborted, skipping the side-load mirror. Fixed separately in +packages/opencoti-llamafile/src/build-pipeline.ts (stream the hash). The DSO link itself holds +under the #629 -mcmodel=large mitigations. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu +@@ -141,24 +141,30 @@ + void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + void ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +-// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620, generalized to the #644 scalar-D128 pair set) — +-// plain (non-DCA) scalar-quant MMA DECODE entries (template-instances/fattn-mma-f16-scalar---d128.cu). +-// Each reads a raw quant K/V cache into the MMA smem tile in-register (load_tile_dequant / get_dequantize_V, +-// no whole-KV materialize), giving quant decode the tensor-core path f16 already gets at high-GQA/long-KV. +-// NOTE: opencoti's CUDA build passes --fa-all-quants (build-pipeline.ts), so every pair below ALSO has a +-// native FA-VEC instance; env=0 keeps that VEC route (kernel-bound ~2x f16 per HAL §3b), env=1 (default) +-// takes this MMA fast-path. Validated q8K/q4V: 70.45 vs f16 60.99 tps (1.15x), 2.30x over native VEC, +-// real_frac=0 (Qwen2.5-14B-1M, 65k). Default/beachhead pair = q8K/q4V (the P1 auto-tier default). +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +-void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620 D128 + #674 D256/D512) — plain (non-DCA) scalar-quant ++// MMA DECODE entries (template-instances/fattn-mma-f16-scalar---d{128,256,512}.cu). Each reads a raw ++// quant K/V cache into the MMA smem tile in-register (load_tile_dequant / get_dequantize_V, no whole-KV ++// materialize), giving quant decode the tensor-core path f16 already gets at high-GQA/long-KV. Head-dims: ++// D128 = Qwen full-attention; D256 = Gemma-4 SWA-local layers; D512 = Gemma-4 global layers (the KV that grows ++// with ctx — where the HAL §3c/§3d −28..35% quant tax bites). NOTE: opencoti's CUDA build passes ++// --fa-all-quants (build-pipeline.ts), so every pair below ALSO has a native FA-VEC instance; env=0 keeps that ++// VEC route (kernel-bound ~2x f16 per HAL §3b), env=1 (default) takes this MMA fast-path. Validated q8K/q4V @ ++// D128: 70.45 vs f16 60.99 tps (1.15x), 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). ++#define OCTI_DECL_SCALAR_MMA(kk, vv) \ ++ void ggml_cuda_flash_attn_ext_mma_f16_scalar_##kk##_##vv##_d128(ggml_backend_cuda_context & ctx, ggml_tensor * dst); \ ++ void ggml_cuda_flash_attn_ext_mma_f16_scalar_##kk##_##vv##_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst); \ ++ void ggml_cuda_flash_attn_ext_mma_f16_scalar_##kk##_##vv##_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst); ++OCTI_DECL_SCALAR_MMA(q8_0, q4_0) ++OCTI_DECL_SCALAR_MMA(q8_0, q8_0) ++OCTI_DECL_SCALAR_MMA(q8_0, q6_0) ++OCTI_DECL_SCALAR_MMA(q8_0, q5_0) ++OCTI_DECL_SCALAR_MMA(q6_0, q6_0) ++OCTI_DECL_SCALAR_MMA(q6_0, q4_0) ++OCTI_DECL_SCALAR_MMA(q5_0, q5_0) ++OCTI_DECL_SCALAR_MMA(q5_1, q5_1) ++OCTI_DECL_SCALAR_MMA(q4_0, q4_0) ++OCTI_DECL_SCALAR_MMA(q4_1, q4_1) ++#undef OCTI_DECL_SCALAR_MMA + + // OPENCOTI_QUANT_MMA_DECODE — DEFAULT-ON; =0 disables the MMA fast-path (each pair reverts to its native + // FA-VEC route). Gates both the decode selector escape (below) and — only for a hypothetical non-all-quants +@@ -170,11 +176,14 @@ + return en; + } + +-// The served DCA-off scalar-MMA decode pairs (mirror of the #644 DCA-on scalar set), D128 only. Both the K!=V +-// guard admittance, the decode selector escape, and the MMA dispatch consult this single predicate so they can +-// never disagree on which pairs are diverted to the tensor-core instances. +-static bool octi_scalar_mma_pair_d128(const ggml_tensor * K, const ggml_tensor * V) { +- if (K->ne[0] != 128 || V->ne[0] != 128) { ++// The served DCA-off scalar-MMA decode pairs (mirror of the #644 DCA-on scalar set) over the three served ++// head-dims: D128 = Qwen full-attention (#620), D256 = Gemma-4 SWA-local layers, D512 = Gemma-4 global layers ++// (#674 — the KV that grows with ctx, where the HAL §3c/§3d −28..35% quant tax bites). Both the K!=V guard ++// admittance, the decode selector escape, and the MMA dispatch consult this single predicate so they can never ++// disagree on which pairs are diverted to the tensor-core instances. Head-dim must be symmetric (K==V) — Gemma ++// enforces DKQ==DV, Qwen is 128/128. ++static bool octi_scalar_mma_pair(const ggml_tensor * K, const ggml_tensor * V) { ++ if (K->ne[0] != V->ne[0] || (K->ne[0] != 128 && K->ne[0] != 256 && K->ne[0] != 512)) { + return false; + } + const ggml_type vt = V->type; +@@ -222,26 +231,32 @@ + if (Q->ne[0] == 512) { ggml_cuda_flash_attn_ext_mma_f16_turbo_q8_0_turbo2_d512(ctx, dst); return; } + } + +- // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — the selector routes a served D128 scalar-K/ +- // scalar-V decode (see octi_scalar_mma_pair_d128) to MMA_F16; hand it to the typed instance (raw in-register +- // quant read) instead of the f16 Q->ne[0] switch below (which would reinterpret the quant blocks as raw +- // f16). Only the served pairs are diverted; all other quant K/V keep the default switch. Guarded by the ++ // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620 D128 + #674 D256/D512) — the selector routes a served ++ // scalar-K/scalar-V decode (see octi_scalar_mma_pair) to MMA_F16; hand it to the head-dim-typed instance (raw ++ // in-register quant read) instead of the f16 Q->ne[0] switch below (which would reinterpret the quant blocks ++ // as raw f16). Only the served pairs are diverted; all other quant K/V keep the default switch. Guarded by the + // same predicate the selector uses — only reached when that selector already returned MMA_F16 for the pair. +- if (octi_quant_mma_decode_enabled() && octi_scalar_mma_pair_d128(K, V)) { ++ // D128 = Qwen full-attn; D256 = Gemma SWA-local; D512 = Gemma global. ++ if (octi_quant_mma_decode_enabled() && octi_scalar_mma_pair(K, V)) { + const ggml_type vt = V->type; +-#define OCTI_SCALAR_MMA_D128(kk, vv, fn) \ +- if (K->type == GGML_TYPE_##kk && vt == GGML_TYPE_##vv) { fn(ctx, dst); return; } +- OCTI_SCALAR_MMA_D128(Q8_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d128) +- OCTI_SCALAR_MMA_D128(Q8_0, Q8_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d128) +- OCTI_SCALAR_MMA_D128(Q8_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d128) +- OCTI_SCALAR_MMA_D128(Q8_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d128) +- OCTI_SCALAR_MMA_D128(Q6_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d128) +- OCTI_SCALAR_MMA_D128(Q6_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d128) +- OCTI_SCALAR_MMA_D128(Q5_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d128) +- OCTI_SCALAR_MMA_D128(Q5_1, Q5_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d128) +- OCTI_SCALAR_MMA_D128(Q4_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d128) +- OCTI_SCALAR_MMA_D128(Q4_1, Q4_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d128) +-#undef OCTI_SCALAR_MMA_D128 ++ const int64_t d = K->ne[0]; ++#define OCTI_SCALAR_MMA(kk, vv, base) \ ++ if (K->type == GGML_TYPE_##kk && vt == GGML_TYPE_##vv) { \ ++ if (d == 128) { base##_d128(ctx, dst); return; } \ ++ if (d == 256) { base##_d256(ctx, dst); return; } \ ++ base##_d512(ctx, dst); return; \ ++ } ++ OCTI_SCALAR_MMA(Q8_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0) ++ OCTI_SCALAR_MMA(Q8_0, Q8_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0) ++ OCTI_SCALAR_MMA(Q8_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0) ++ OCTI_SCALAR_MMA(Q8_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0) ++ OCTI_SCALAR_MMA(Q6_0, Q6_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0) ++ OCTI_SCALAR_MMA(Q6_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0) ++ OCTI_SCALAR_MMA(Q5_0, Q5_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0) ++ OCTI_SCALAR_MMA(Q5_1, Q5_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1) ++ OCTI_SCALAR_MMA(Q4_0, Q4_0, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0) ++ OCTI_SCALAR_MMA(Q4_1, Q4_1, ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1) ++#undef OCTI_SCALAR_MMA + } + + switch (Q->ne[0]) { +@@ -641,7 +656,7 @@ + (V->type == GGML_TYPE_TURBO2_0 || V->type == GGML_TYPE_TURBO3_0 || + V->type == GGML_TYPE_TURBO2_TCQ || V->type == GGML_TYPE_TURBO3_TCQ); + // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — admit the served asymmetric scalar pairs +- // (octi_scalar_mma_pair_d128, e.g. q8_0-K/q4_0-V) so the selector below can route their decode to the ++ // (octi_scalar_mma_pair, e.g. q8_0-K/q4_0-V) so the selector below can route their decode to the + // scalar-MMA instance. This whole block is #ifndef GGML_CUDA_FA_ALL_QUANTS, so under opencoti's actual + // --fa-all-quants build it is COMPILED OUT (every pair already has a native FA-VEC instance and reaches + // the selector); this admittance is the fallback that keeps the feature working on a hypothetical +@@ -650,7 +665,7 @@ + // (no scalar-MMA block_sel path) — keep the pair UNSUPPORTED in that combo (graceful NONE) rather than + // admitting it into a dispatch with no matching kernel. + const bool scalar_k_scalar_v_mma = +- octi_quant_mma_decode_enabled() && octi_scalar_mma_pair_d128(K, V) && dst->src[7] == nullptr; ++ octi_quant_mma_decode_enabled() && octi_scalar_mma_pair(K, V) && dst->src[7] == nullptr; + if (K->type != V->type && !both_tcq && !scalar_k_turbo_v && !scalar_k_scalar_v_mma) { + return BEST_FATTN_KERNEL_NONE; + } +@@ -758,8 +773,8 @@ + // If Turing tensor cores are available, use them: + if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if (can_use_vector_kernel) { +- // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620) — for a served scalar pair @ D128 +- // (octi_scalar_mma_pair_d128, default-on) skip the ENTIRE VEC-decision so it falls straight to MMA ++ // opencoti-hook: #582-P0 DCA-off quant-MMA decode (#620 D128 + #674 D256/D512) — for a served scalar ++ // pair (octi_scalar_mma_pair, default-on) skip the ENTIRE VEC-decision so it falls straight to MMA + // below, for every n_q and every n_kv (prefill, decode, short- or long-context). Unlike f16 (which + // keeps its VEC decode path and only escapes to MMA at high-GQA/long-KV), quant reclaim is + // unconditional for these pairs — the whole point is to take quant decode OFF the kernel-bound +@@ -767,7 +782,7 @@ + // route (opencoti builds --fa-all-quants, so that VEC instance exists — no crash, just slower). + // dst->src[7] guard: sparse-attn (vslash) keeps the FA-VEC block_sel reader (no scalar-MMA block_sel). + const bool octi_scalar_mma = octi_quant_mma_decode_enabled() +- && octi_scalar_mma_pair_d128(K, V) && dst->src[7] == nullptr; ++ && octi_scalar_mma_pair(K, V) && dst->src[7] == nullptr; + if (!octi_scalar_mma) { + if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { + // opencoti bug-858 (2026-07-05): REVERTED the earlier <=2 experiment back to upstream's +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. The shared mechanism was validated at D128: q8_0/q4_0 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q4_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q4_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. The shared mechanism was validated at D128: q8_0/q4_0 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q4_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q4_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q4_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q8_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q8_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q8_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q8_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q8_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q8_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q8_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q8_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q8_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q6_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q6_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q6_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q6_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q6_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q5_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q8_0-q5_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q8_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q8_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q8_0_q5_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q8_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q8_0_q5_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q8_0_q5_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q6_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q6_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q6_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q6_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q6_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q6_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q6_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q6_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q4_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q6_0-q4_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q6_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q6_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q6_0_q4_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q6_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q6_0_q4_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q6_0_q4_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_0_q5_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_0-q5_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_0-K / Q5_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_0/q5_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_0_q5_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q5_0, GGML_TYPE_Q5_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_0_q5_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_0_q5_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_1-K / Q5_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_1/q5_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_1_q5_1_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q5_1-q5_1-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q5_1-K / Q5_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q5_1/q5_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q5_1_q5_1_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q5_1, GGML_TYPE_Q5_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q5_1_q5_1_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q5_1_q5_1_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_0_q4_0_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_0-q4_0-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_0-K / Q4_0-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_0/q4_0 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_0_q4_0_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_0_q4_0_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_0_q4_0_d512_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d256.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d256.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d256.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D256 generalization of the #620 D128 beachhead) — D256 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_1-K / Q4_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_1/q4_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D256 = Gemma-4 SWA-local head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==256 (full gqa 1/2/4/8), matching the turbo-MMA D256 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_1_q4_1_d256_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 16/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 32/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<256, 256, 64/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D256 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d256_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d256_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d256_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d256_ncols1<1>(ctx, dst); ++} +diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d512.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d512.cu +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-scalar-q4_1-q4_1-d512.cu +@@ -0,0 +1,76 @@ ++// opencoti-hook: #582-P0 DCA-off quant-MMA decode (#674 — D512 generalization of the #620 D128 beachhead) — D512 plain (non-DCA) scalar-quant ++// tensor-core DECODE kernel for a Q4_1-K / Q4_1-V cache. opencoti's CUDA build passes --fa-all-quants, so a ++// native FA-VEC instance for this pair EXISTS; HAL §3b measured that VEC reader as kernel-bound (~2x f16, ++// regardless of bytes moved). f16 at high-GQA/long-KV already falls to the tensor-core MMA path (fattn.cu ++// selector), so this TU gives the scalar cache the same route: it reads the raw q4_1/q4_1 blocks into the ++// MMA smem tile in-register via load_tile_dequant / get_dequantize_V (no whole-KV materialize), then runs the ++// tensor-core mul_mat. Scalar KV is NOT FWHT-rotated, so there is no wht_o / de-rotate (unlike the turbo ++// verify kernel #630). Routed by the CUDA selector (fattn.cu) only when OPENCOTI_QUANT_MMA_DECODE != 0 ++// (default-on); env=0 reverts the pair to its native FA-VEC path. Validated pair q8_0/q4_0: 70.45 vs f16 ++// 60.99 tps (1.15x) and 2.30x over native VEC, real_frac=0 (Qwen2.5-14B-1M, 65k). D512 = Gemma-4 global head-dim; correctness gated by the #674 A4B logit-equiv/tps gate. New additive TU, ++// auto-collected by build-functions.sh's fattn-mma-*.cu glob (#531); ncols selection mirrors the f16 ++// switch_ncols2/ncols1 for DKQ==DV==512 (full gqa 1/2/4/8), matching the turbo-MMA D512 sibling (#630). ++ ++#include "../fattn-mma-f16.cuh" ++ ++template ++static void ggml_cuda_fa_mma_scalar_q4_1_q4_1_d512_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; ++ const ggml_tensor * Q = dst->src[0]; ++ ++ if constexpr (ncols2 <= 16) { ++ if (Q->ne[1] <= 16/ncols2) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 16/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ } ++ ++ if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING)) { ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 32/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++ return; ++ } ++ ++ ggml_cuda_flash_attn_ext_mma_f16_case<512, 512, 64/ncols2, ncols2, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1>(ctx, dst); ++} ++ ++// Entry invoked from fattn.cu's MMA dispatch when the selector routes a D512 scalar-K/scalar-V decode. ++void ggml_cuda_flash_attn_ext_mma_f16_scalar_q4_1_q4_1_d512(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ++ const ggml_tensor * KQV = dst; ++ const ggml_tensor * Q = dst->src[0]; ++ const ggml_tensor * K = dst->src[1]; ++ const ggml_tensor * V = dst->src[2]; ++ const ggml_tensor * mask = dst->src[3]; ++ ++ float max_bias = 0.0f; ++ memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); ++ ++ bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; ++ for (const ggml_tensor * t : {Q, K, V, mask}) { ++ if (t == nullptr || ggml_is_quantized(t->type)) { ++ continue; ++ } ++ for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { ++ if (t->nb[i] % 16 != 0) { ++ use_gqa_opt = false; ++ break; ++ } ++ } ++ } ++ ++ GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); ++ const int gqa_ratio = Q->ne[2] / K->ne[2]; ++ ++ if (use_gqa_opt && gqa_ratio > 4) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d512_ncols1<8>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 2) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d512_ncols1<4>(ctx, dst); ++ return; ++ } ++ if (use_gqa_opt && gqa_ratio > 1) { ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d512_ncols1<2>(ctx, dst); ++ return; ++ } ++ ggml_cuda_fa_mma_scalar_q4_1_q4_1_d512_ncols1<1>(ctx, dst); ++} diff --git a/patches/0128-mtp-spec-clone-cheap-mmvf-verify.patch b/patches/0128-mtp-spec-clone-cheap-mmvf-verify.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca5bf516bd3134209e1df6d2fdcbaad08b69a66f --- /dev/null +++ b/patches/0128-mtp-spec-clone-cheap-mmvf-verify.patch @@ -0,0 +1,119 @@ +opencoti patch 0128 — MTP spec-cycle host parity: spec-clone-cheap + mmvf f16/bf16 verify lift (bug-858 / #675) + +Closes the last Qwen NextN speculative-decode throughput gap vs upstream b9859: +−12% → −1.7% at spec depth 3 (bs2 GPU1, Qwen3.6-35B-A3B UD-Q6_K, n3 282.6 vs +287.4 t/s; base decode 212.6 = parity, unregressed). Two independent fixes, one +GPU-side and one host-side; output is bit-identical pre/post on greedy (accept +0.667 unchanged) — both are pure-latency fixes. + +1) mmvf.cu f16/bf16 ampere_mma TinyBLAS lift (GPU; bug-858/#675) + Sibling of 0093's f32 lift (bug-2103/#609). Upstream caps the ampere_mma + f16/bf16 mmvf branches at ne11==1 because past that its cuBLAS fallback + wins; llamafile's TinyBLAS build has no cuBLAS, so the fallback is a + single-block tinyblasGE cliff. In an MTP verify batch (ne11 = n_accepted+1 + = 2..4) the thin shared-expert gate GEMVs (bf16 blk.N.ffn_gate_inp_shexp, + {K=2048, M=1}) take that cliff every cycle — the dominant residual + tinyblasGE cost after #609 fixed the f32 routers. Fix: `#ifdef + GGML_USE_TINYBLAS return src0_small && ne11 <= 8;` in both branches + (`src0_small` preserves the upstream thin-matrix guard; base decode ne11==1 + unchanged; cuBLAS builds unchanged). Marker: `opencoti-hook: + mtp-verify-mmvf (bug-858/#675)` (2 sites; joins 0093's f32 site). + +2) common/sampling.cpp spec-clone-cheap (HOST; the actual wall-clock gap) + `common_sampler_clone` deep-copied the n_vocab-sized `cur` scratch + (~1.8 MiB) every speculative cycle. Under the cosmo dlmalloc runtime every + ≥256 KiB allocation is a fresh mmap + page-fault-on-write + munmap — + measured 1.071 ms/cycle (per-phase OPENCOTI_SPEC_PROF instrumentation, + process→sample gap), i.e. the ENTIRE spec wall-clock deficit vs b9859, + whose glibc reuses the block for ~0.07 ms. Identical source, runtime- + allocator asymmetry — invisible to nsys/GPU profiling (constant host time). + Fix: clone with empty `cur`/`cur_p`. `cur` is write-only scratch that + `set_logits()` repopulates before any read, and the copied `cur_p.data` + pointed into the SOURCE sampler's buffer anyway (latent upstream aliasing), + so no consumer can rely on the clone's copy; restore via move-assign is + unaffected. Clone cost 1.071 → 0.005 ms/cycle. Marker: `opencoti-hook: + spec-clone-cheap (bug-858/#675)`. + +GATE (bs2 GPU1, clean binary 05240e8e57c7191e, all diagnostics stripped): +base 212.59 (parity vs b9859 212.1); Qwen n3 282.63 vs b9859 287.39 (−1.7%, +was −12%); 512-token cells n1/n2/n3 255.5/260.5/265.2; accept bit-identical +pre/post fix. Full re-validated MTP matrix (all 4 documented pairs, ours vs +b9859 same-day/same-GPU) + 9-prompt am17an bench: docs/evaluations/ +three-way-tps.md "2026-07-14 refresh". Residuals (characterized, NOT this +patch): accept deficit at depth ≥2 (#495-class chained-draft numerics) and +~0.15-0.2 ms/cycle pre-draft glue. + +Build note: host-only rebuild suffices for (2); (1) needs the CUDA DSO. The +capture session also hit bug-2181 (ccache+cosmocc fat-object: a cache hit +restores only the x86_64 .o, leaving a stale .aarch64 sidecar packed into the +APE) — after reverting diagnostics, delete both .o twins and rebuild with +CCACHE_DISABLE=1, verify with `strings` on the APE. + +Provenance: opencoti-original. Applies after `0127`. + +diff --git a/llama.cpp/ggml/src/ggml-cuda/mmvf.cu b/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +index 4051188..6eca59d 100644 +--- a/llama.cpp/ggml/src/ggml-cuda/mmvf.cu ++++ b/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +@@ -826,6 +826,19 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 + if (GGML_CUDA_CC_IS_NVIDIA(cc)) { + const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1); + if (ampere_mma_available(cc)) { ++#ifdef GGML_USE_TINYBLAS ++ // opencoti-hook: mtp-verify-mmvf (bug-858/#675) — sibling of the f32 lift ++ // above. Upstream caps ampere_mma f16/bf16 at ne11==1 because past that its ++ // cuBLAS fallback wins; TinyBLAS has no cuBLAS and the fallback is a ++ // single-block tinyblasGE cliff. In an MTP verify batch (ne11 = n_accepted+1 ++ // = 2..4) the thin-src0 shared-expert gate ({K,M=1}, e.g. ffn_gate_inp_shexp) ++ // takes that cliff — the dominant residual tinyblasGE cost after #609 fixed ++ // the f32 routers. Keep it on the mmvf kernel (its real small-src0 max is the ++ // ADA branch's 4; use 8 to match the non-mma default). src0_small preserves ++ // the upstream guard so only thin matrices are lifted. Base decode (ne11==1) ++ // is unchanged. Remove/re-gate this hook if TinyBLAS is ever replaced. ++ return src0_small && ne11 <= 8; ++#endif // GGML_USE_TINYBLAS + return src0_small && ne11 == 1; + } + if (cc >= GGML_CUDA_CC_ADA_LOVELACE) { +@@ -852,6 +865,13 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 + if (GGML_CUDA_CC_IS_NVIDIA(cc)) { + const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1); + if (ampere_mma_available(cc)) { ++#ifdef GGML_USE_TINYBLAS ++ // opencoti-hook: mtp-verify-mmvf (bug-858/#675) — bf16 sibling of the f32/f16 ++ // lifts. This is the exact tensor the #675 trace caught cliffing to ++ // tinyblasGE: blk.N.ffn_gate_inp_shexp.weight (bf16, K=2048, M=1) at verify ++ // ncols 2..4. See the f16 case above for the full rationale. ++ return src0_small && ne11 <= 8; ++#endif // GGML_USE_TINYBLAS + return src0_small && ne11 == 1; + } + if (cc >= GGML_CUDA_CC_ADA_LOVELACE) { +diff --git a/llama.cpp/common/sampling.cpp b/llama.cpp/common/sampling.cpp +index 5665d0a..ad93fdf 100644 +--- a/llama.cpp/common/sampling.cpp ++++ b/llama.cpp/common/sampling.cpp +@@ -470,14 +470,21 @@ void common_sampler_reset(struct common_sampler * gsmpl) { + } + + struct common_sampler * common_sampler_clone(common_sampler * gsmpl) { ++ // opencoti-hook: spec-clone-cheap (bug-858/#675) — do NOT deep-copy the n_vocab-sized ++ // `cur` scratch. Under the cosmo dlmalloc runtime the ~1.8 MiB copy is a fresh mmap + ++ // page-fault-on-write + munmap on EVERY speculative cycle (measured 1.07 ms/cycle on ++ // Qwen3.6-35B NextN — the entire spec wall-clock gap vs upstream b9859, whose glibc ++ // reuses the block for ~free). `cur` is write-only scratch: set_logits() repopulates it ++ // before any read, and the copied `cur_p.data` pointed into the SOURCE's buffer anyway, ++ // so no consumer can rely on the clone's copy. Restore via move-assign is unaffected. + return new common_sampler { + /* .params = */ gsmpl->params, + /* .grmr = */ llama_sampler_clone(gsmpl->grmr), + /* .rbudget = */ llama_sampler_clone(gsmpl->rbudget), + /* .chain = */ llama_sampler_clone(gsmpl->chain), + /* .prev = */ gsmpl->prev, +- /* .cur = */ gsmpl->cur, +- /* .cur_p = */ gsmpl->cur_p, ++ /* .cur = */ {}, ++ /* .cur_p = */ {}, + }; + } + diff --git a/patches/0129-rys-probe-nested-args.patch b/patches/0129-rys-probe-nested-args.patch new file mode 100644 index 0000000000000000000000000000000000000000..fb06657c027db2257acdd668566de5fa150ee8ab --- /dev/null +++ b/patches/0129-rys-probe-nested-args.patch @@ -0,0 +1,76 @@ +opencoti patch 0129 — rys-probe nested-tree params + CLI flags (capture gap of 0123 / #669) + +Completes the RYS-probe (`--rys-probe`, docs/features/rys_probe.md) capture: +patch 0123 landed only the OUTER llamafile/ half (args.cpp, rys_probe.cpp, +BUILD.mk, main.cpp), but the feature also added the common_params fields +(rys_probe, rys_probe_widths, rys_probe_band, rys_probe_topk) to +common/common.h and the four `--rys-probe*` add_opt registrations to +common/arg.cpp in the NESTED llama.cpp tree. Those two files were live in the +working tree but missing from the patch chain, so a fresh +`build:llamafile:apply` would produce a tree that fails to compile the outer +rys_probe.cpp (unknown common_params members). No behavior change vs the live +tree — this is purely a chain-integrity capture. + +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +index 958e0db..0bea0d2 100644 +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -1552,6 +1552,40 @@ common_params_context common_params_parser_init(common_params & params, llama_ex + params.repeat_layers = value; // parsed + validated at load (llama-context ctor) + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REPEAT_LAYERS")); ++ // opencoti RYS probe (`--rys-probe`) — shortlist generator for RYS templates — see docs/features/rys_probe.md ++ add_opt(common_arg( ++ {"--rys-probe"}, ++ "RYS template PROBE mode: enumerate contiguous mid-stack layer-duplication blocks, score each by " ++ "ΔPPL (on -f/-p corpus, else a small built-in) + a small arithmetic task-probe, and print a " ++ "ΔPPL-vs-overhead Pareto shortlist + the best --repeat-layers spec. Loads the model once; host-only, " ++ "no CUDA. Shortlist NOT a verdict — validate the top picks on your real harness.", ++ [](common_params & params) { ++ params.rys_probe = true; ++ } ++ ).set_examples({LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); ++ add_opt(common_arg( ++ {"--rys-probe-widths"}, "auto|W1,W2,...", ++ "RYS probe: block widths to enumerate. 'auto' (default) = every width in the band " ++ "(so both a cheap and a max-gain template can be found); or an explicit list like 2,4,8.", ++ [](common_params & params, const std::string & value) { ++ params.rys_probe_widths = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); ++ add_opt(common_arg( ++ {"--rys-probe-band"}, "auto|i:j", ++ "RYS probe: candidate layer band (default: auto = [max(3,n/8) .. n-max(3,n/8)), the mid-stack " ++ "boundary-safe band). `i:j` overrides it.", ++ [](common_params & params, const std::string & value) { ++ params.rys_probe_band = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); ++ add_opt(common_arg( ++ {"--rys-probe-topk"}, "N", ++ "RYS probe: how many ranked candidates to print (default: 10).", ++ [](common_params & params, int value) { ++ params.rys_probe_topk = value; ++ } ++ ).set_examples({LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + // opencoti #551 sparse-attn — Quest-style block-selector sparse attention — see docs/features/sparse_attn.md + add_opt(common_arg( + {"--sparse-attn"}, "MODE", +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index d44d51d..f2e2f17 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -490,6 +490,13 @@ struct common_params { + // with SHARED weights (no extra param VRAM). "" = off (byte-identical). Dense Qwen3 only in the + // MVP; hard-errors at load on iSWA/hybrid/DCA/sparse. See docs/features/rys_layer_duplication.md. + std::string repeat_layers; ++ // opencoti RYS probe (`--rys-probe`, docs/features/rys_probe.md) — shortlist ++ // generator for RYS layer-duplication templates. Loads the model once and ++ // scores contiguous mid-stack blocks by ΔPPL + a small task-probe. Host-only. ++ bool rys_probe = false; ++ std::string rys_probe_widths = "auto"; // block widths: "auto" = all in band, or W1,W2,... ++ std::string rys_probe_band = "auto"; // candidate band: auto | i:j ++ int32_t rys_probe_topk = 10; // rows in the context table + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / diff --git a/patches/0130-parallel2-fit-ckpt-fixes.patch b/patches/0130-parallel2-fit-ckpt-fixes.patch new file mode 100644 index 0000000000000000000000000000000000000000..bffc36766de7b07282dd53ba85f588affc70725f --- /dev/null +++ b/patches/0130-parallel2-fit-ckpt-fixes.patch @@ -0,0 +1,322 @@ +opencoti patch 0130 — parallel≥2 boot + checkpoint-restore robustness (bug-2182, bug-2183) + +Two independent server-robustness fixes, both flushed out by the first +`--parallel 2` gate on quant KV (bs2 GPU1, A4B v7-coder Q8_0, q8_0 KV): + +1) bug-2182 — `ggml_can_repeat` GGML_ASSERT abort at `-fit` / graph build + (src/llama-kv-cache.cpp). The opencoti per-stream / windowed KV scatter + accumulates dependency-tie handles by adding one-block views cast to F32; + the operand's ne was the blck_size of the SOURCE tensor (32 for + q8_0/turbo, 1 for f16). With n_stream ≥ 2 the tie chain is already an F32 + [1] tensor when the next quant store's [32] operand is added → + add(a=[1], b=[32]) fails ggml_can_repeat → abort. Fires only with quant + KV × parallel ≥ 2 non-unified when the scatter branch engages (the `-fit` + trial engages the rolling-KV window branch even when the live boot is + fully resident — why "fitting params" is in every crash signature). Same + class as bug-2170 (0125, recurrent hybrids). Fix: static helper + `tie_scalar_f32()` — view ONE FULL BLOCK (legal for any quant type), cast + to F32 (blck 1), take a single element — so every tie operand is exactly + one F32 element regardless of source type; all 20 dep-tie sites in + cpy_k/cpy_v (M2 scatter, hybrid order-agnostic acc2, append-only windowed + accumulate/emit) rewritten to use it. Byte-identical graphs for f16/f32 + KV (blck 1 before and after). + +2) bug-2183 — hard GGML_ABORT on failed context-checkpoint restore + (common/common.h, common/common.cpp, tools/server/server-context.cpp). + `common_prompt_checkpoint::load_tgt/load_dft` aborted the whole server + when `llama_state_seq_set_data_ext` returned 0 — which legitimately + happens under `--kv-unified --parallel ≥ 2` when the unified KV cannot + allocate cells while another slot is mid-decode. Fix: the loaders return + bool (WARN + false instead of abort); the server task-launch restore site + handles failure by wiping the seq (tgt + dft) and falling back to full + prompt re-processing via the existing do_reset path. The two speculative + rollback/draft sites keep fail-fast GGML_ABORTs (a silent continue there + would corrupt decode state). + +Validated on the 3090 with the exact bs2 crash configs: (1) A4B v7 Q4_K_M, +q8_0 KV, --parallel 2 non-unified, -fit on → boots n_slots=2 and decodes; +(2) forced `state_read_meta: failed to find available cells` live under +kv-unified parallel-2 with prompt cache on → 0 aborts, all requests +completed with the FAILED→re-process fallback. The parser655 workarounds +(`-fit off -cram 0 -ctxcp 0`) are no longer needed for boot. + +diff --git a/llama.cpp/common/common.cpp b/llama.cpp/common/common.cpp +index f149e29..9e6aa55 100644 +--- a/llama.cpp/common/common.cpp ++++ b/llama.cpp/common/common.cpp +@@ -2198,40 +2198,50 @@ void common_prompt_checkpoint::update_dft( + } + } + +-void common_prompt_checkpoint::load_tgt( ++bool common_prompt_checkpoint::load_tgt( + llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags) const { + if (ctx == nullptr) { +- return; ++ return true; + } + + if (data_tgt.empty()) { +- return; ++ return true; + } + + const size_t n = llama_state_seq_set_data_ext(ctx, data_tgt.data(), data_tgt.size(), seq_id, flags); + if (n != data_tgt.size()) { +- GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", data_tgt.size(), n); ++ // opencoti bug-2183: not fatal — the KV cache may be unable to allocate ++ // cells right now (e.g. kv-unified with another slot mid-decode) ++ LOG_WRN("%s: checkpoint restore failed: expected %zu, got %zu\n", __func__, data_tgt.size(), n); ++ return false; + } ++ ++ return true; + } + +-void common_prompt_checkpoint::load_dft( ++bool common_prompt_checkpoint::load_dft( + llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags) const { + if (ctx == nullptr) { +- return; ++ return true; + } + + if (data_dft.empty()) { +- return; ++ return true; + } + + const size_t n = llama_state_seq_set_data_ext(ctx, data_dft.data(), data_dft.size(), seq_id, flags); + if (n != data_dft.size()) { +- GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", data_dft.size(), n); ++ // opencoti bug-2183: not fatal — the KV cache may be unable to allocate ++ // cells right now (e.g. kv-unified with another slot mid-decode) ++ LOG_WRN("%s: checkpoint restore failed: expected %zu, got %zu\n", __func__, data_dft.size(), n); ++ return false; + } ++ ++ return true; + } + + void common_prompt_checkpoint::clear_tgt() { +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index f2e2f17..66c5ac6 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -1180,12 +1180,15 @@ struct common_prompt_checkpoint { + llama_seq_id seq_id, + llama_state_seq_flags flags); + +- void load_tgt( ++ // opencoti bug-2183: restore can fail (state_seq_set_data returns 0) when the ++ // unified KV cannot allocate cells while another slot is mid-decode; callers ++ // must handle false (fall back to full re-processing) instead of dying. ++ bool load_tgt( + llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags) const; + +- void load_dft( ++ bool load_dft( + llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags) const; +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 7057531..eb5899f 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -3966,6 +3966,18 @@ uint32_t llama_kv_cache::headinfer_window_cells(int32_t il) const { + return layers[it->second].window_cells; + } + ++// opencoti bug-2182: dependency-tie scalar handle. The dep-tie pattern adds ++// one-block views of two stores to force graph ordering; the add operands were ++// sized by each tensor's BLOCK size (32 for q8_0/turbo, 1 for f16/f32), so a ++// mixed pair (F32 tie chain + quant store, reached at n_stream >= 2) failed ++// ggml_can_repeat and aborted at graph build (fit trial / reserve). Collapse ++// every tie operand to exactly one F32 element: view one full block (legal for ++// any quant type), cast to F32 (blck 1), then take a single element. ++static ggml_tensor * tie_scalar_f32(ggml_context * ctx, ggml_tensor * t) { ++ ggml_tensor * f = ggml_cast(ctx, ggml_view_1d(ctx, t, ggml_blck_size(t->type), 0), GGML_TYPE_F32); ++ return ggml_view_1d(ctx, f, 1, 0); ++} ++ + ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { + const int32_t ikv = mli_at_checked(map_layer_ids, il, __func__); + +@@ -4029,14 +4041,14 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + ggml_tensor * store_g = ggml_set_rows(ctx, k_s, cur_g, idxs_s); + ggml_tensor * store_c = ggml_set_rows(ctx, kc_s, cur_c, idxs_s); + ggml_tensor * pair = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, store_g), ++ tie_scalar_f32(ctx, store_c)); + if (!tie) { + tie = pair; + } else { + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, pair)); + } + } + return tie; +@@ -4081,8 +4093,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + auto acc2 = [&](ggml_tensor * store) { + if (!tie2) { tie2 = store; return; } + tie2 = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie2, ggml_blck_size(tie2->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie2), ++ tie_scalar_f32(ctx, store)); + }; + for (uint32_t s = 0; s < batch_ns; ++s) { + ggml_tensor * k_s = layer.k_per_stream[sinfo.strm[s]]; +@@ -4104,8 +4116,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + auto accumulate = [&](ggml_tensor * store) { + if (!tie) { tie = store; return; } + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, store)); + }; + auto emit = [&](ggml_tensor * dst, int64_t row0, int64_t nrows, size_t base) { + if (nrows <= 0) return; +@@ -4179,8 +4191,8 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm + tie = store_s; + } else { + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, store_s)); + } + } + return tie; +@@ -4239,14 +4251,14 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + ggml_tensor * store_g = ggml_set_rows(ctx, v_s, cur_g, idxs_s); + ggml_tensor * store_c = ggml_set_rows(ctx, vc_s, cur_c, idxs_s); + ggml_tensor * pair = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, store_g, ggml_blck_size(store_g->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store_c, ggml_blck_size(store_c->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, store_g), ++ tie_scalar_f32(ctx, store_c)); + if (!tie) { + tie = pair; + } else { + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, pair, ggml_blck_size(pair->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, pair)); + } + } + return tie; +@@ -4288,8 +4300,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + auto acc2 = [&](ggml_tensor * store) { + if (!tie2) { tie2 = store; return; } + tie2 = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie2, ggml_blck_size(tie2->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie2), ++ tie_scalar_f32(ctx, store)); + }; + for (uint32_t s = 0; s < batch_ns; ++s) { + ggml_tensor * v_s = layer.v_per_stream[sinfo.strm[s]]; +@@ -4311,8 +4323,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + auto accumulate = [&](ggml_tensor * store) { + if (!tie) { tie = store; return; } + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store, ggml_blck_size(store->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, store)); + }; + auto emit = [&](ggml_tensor * dst, int64_t row0, int64_t nrows, size_t base) { + if (nrows <= 0) return; +@@ -4363,8 +4375,8 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm + tie = store_s; + } else { + tie = ggml_add(ctx, +- ggml_cast(ctx, ggml_view_1d(ctx, tie, ggml_blck_size(tie->type), 0), GGML_TYPE_F32), +- ggml_cast(ctx, ggml_view_1d(ctx, store_s, ggml_blck_size(store_s->type), 0), GGML_TYPE_F32)); ++ tie_scalar_f32(ctx, tie), ++ tie_scalar_f32(ctx, store_s)); + } + } + return tie; +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 04207b2..3975235 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -2715,7 +2715,9 @@ private: + + if (ctx_dft) { + if (use_ckpt_dft) { +- ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); ++ if (!ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE)) { ++ GGML_ABORT("speculative draft checkpoint restore failed\n"); ++ } + } + + common_context_seq_rm(ctx_dft.get(), slot.id, ckpt.pos_max + 1, -1); +@@ -3011,12 +3013,26 @@ private: + + if (!do_reset) { + // restore the context checkpoint +- it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); +- it->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); +- +- pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); +- n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); +- SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, n_past, (float) it->size() / 1024 / 1024); ++ // opencoti bug-2183: the restore can fail (state_seq_set_data == 0) when the ++ // unified KV cannot find cells while another slot is mid-decode. A partial ++ // restore may have left junk in this seq, so wipe it and fall back to full ++ // prompt re-processing (the do_reset path) instead of aborting the server. ++ const bool ok_ckpt = ++ it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) && ++ it->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ ++ if (ok_ckpt) { ++ pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max)); ++ n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens); ++ SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_past = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, n_past, (float) it->size() / 1024 / 1024); ++ } else { ++ SLT_WRN(slot, "context checkpoint restore FAILED (pos_min = %d, pos_max = %d) — wiping seq and re-processing the full prompt\n", it->pos_min, it->pos_max); ++ llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot.id, -1, -1); ++ if (ctx_dft) { ++ llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, -1, -1); ++ } ++ do_reset = true; ++ } + } + + if (do_reset) { +@@ -3641,13 +3657,17 @@ private: + SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); + + { +- ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); ++ if (!ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE)) { ++ GGML_ABORT("speculative rollback checkpoint restore failed (tgt)\n"); ++ } + + common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1); + } + + if (slot.ctx_dft) { +- ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); ++ if (!ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE)) { ++ GGML_ABORT("speculative rollback checkpoint restore failed (dft)\n"); ++ } + + common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1); + } diff --git a/patches/0131-rys-probe-task-battery-v2.patch b/patches/0131-rys-probe-task-battery-v2.patch new file mode 100644 index 0000000000000000000000000000000000000000..3858bc6e19ca8fb4bd5373d6bd1ebc0f18d387ba --- /dev/null +++ b/patches/0131-rys-probe-task-battery-v2.patch @@ -0,0 +1,430 @@ +diff --git a/llamafile/rys_probe.cpp b/llamafile/rys_probe.cpp +index 5064fbf5b..609090b07 100644 +--- a/llamafile/rys_probe.cpp ++++ b/llamafile/rys_probe.cpp +@@ -6,12 +6,18 @@ + // Shortlist generator for RYS layer-duplication templates (`--repeat-layers`). + // Loads the model ONCE, then for each candidate contiguous mid-stack block + // rebuilds only the llama_context with a mutated repeat_layers plan and scores +-// it by (a) ΔPPL on a small corpus and (b) a small arithmetic/short-reasoning +-// task-probe. Prints a ΔPPL-vs-overhead Pareto shortlist + the best +-// --repeat-layers spec to paste. Host-only; no CUDA, no new kernels. ++// it. The ranking signal is CAPABILITY, not perplexity: layer duplication costs ++// decode tps (extra effective layers + KV), so a template only earns its keep if ++// it measurably RAISES a balanced task battery over base. Primary = Δtask-vs-base ++// (arithmetic + multi-step reasoning + AIME-style competition + coding output- ++// prediction + factual). PPL is a SECONDARY signal only (reported + tiebreak). ++// Prints a Δtask-ranked shortlist + the best --repeat-layers spec to paste. ++// Host-only; no CUDA, no new kernels. + // +-// This is a *shortlist*, not a verdict: PPL is a weak proxy for reasoning gains +-// (dnhkng RYS-II used graded Math/EQ). Validate the top-K on the real harness. ++// This is a *shortlist*, not a verdict: the battery is small and greedy-graded. ++// If NO block raises the battery, the probe says so plainly — that is the honest ++// answer (duplication not worth its tps cost here). Validate winners on the real ++// harness (perf/llamafile/rys-probe-eval.sh; dnhkng RYS-II used graded Math/EQ). + + #include "common.h" + #include "arg.h" +@@ -19,6 +25,7 @@ + #include "llama.h" + + #include ++#include + #include + #include + #include +@@ -29,48 +36,84 @@ namespace lf { + + namespace { + ++// Balanced task battery: 5 categories so the score reflects broad capability, ++// not one skill. Items are deliberately HARDER than trivial recall so base does ++// not saturate at 1.0 — a saturated probe has no headroom to show a gain, which ++// was the flaw of the old ΔPPL-ranked version. Greedy decode, boundary-aware ++// answer match (see answer_hit), early-stop on hit. ++enum TaskCat { CAT_ARITH = 0, CAT_REASON, CAT_AIME, CAT_CODE, CAT_FACT, N_CAT }; ++const char * kCatName[N_CAT] = { "arith", "reason", "aime", "code", "fact" }; ++ ++// Per-category default generation budget (tokens). Reasoning/AIME get more room ++// for a short greedy chain-of-thought before the answer; recall/arith are terse. ++const int kCatGen[N_CAT] = { 96, 192, 256, 96, 48 }; ++ ++struct TaskQA { TaskCat cat; const char * q; const char * a; }; ++ ++const TaskQA kTasks[] = { ++ // --- arithmetic (large products / factorials / series; harder for headroom) --- ++ {CAT_ARITH, "Q: What is 23 times 47?\nA:", "1081"}, ++ {CAT_ARITH, "Q: What is 156 times 23?\nA:", "3588"}, ++ {CAT_ARITH, "Q: What is 12345 plus 67890?\nA:", "80235"}, ++ {CAT_ARITH, "Q: What is 2 to the power of 15?\nA:", "32768"}, ++ {CAT_ARITH, "Q: What is the sum of all integers from 1 to 50?\nA:", "1275"}, ++ {CAT_ARITH, "Q: What is 7 factorial?\nA:", "5040"}, ++ {CAT_ARITH, "Q: What is 1000 minus 347 minus 158?\nA:", "495"}, ++ {CAT_ARITH, "Q: What is 45 times 45?\nA:", "2025"}, ++ {CAT_ARITH, "Q: What is the remainder when 1000 is divided by 13?\nA:", "12"}, ++ // --- multi-step word problems / reasoning (small integer answers) --- ++ {CAT_REASON, "Q: A number increased by 20 percent equals 60. What is the number?\nA:", "50"}, ++ {CAT_REASON, "Q: The sum of three consecutive integers is 72. What is the largest?\nA:", "25"}, ++ {CAT_REASON, "Q: A rectangle has area 48 and width 6. What is its perimeter?\nA:", "28"}, ++ {CAT_REASON, "Q: If 5 workers build a wall in 12 days, how many days do 3 workers need?\nA:", "20"}, ++ {CAT_REASON, "Q: What is the units digit of 7 to the power of 4?\nA:", "1"}, ++ {CAT_REASON, "Q: How many positive divisors does 12 have?\nA:", "6"}, ++ {CAT_REASON, "Q: The average of four numbers is 15. Three of them are 10, 20, and 12. What is the fourth?\nA:", "18"}, ++ {CAT_REASON, "Q: How many two-digit numbers are divisible by 9?\nA:", "10"}, ++ // --- AIME-style competition (harder; expect real headroom / floor) --- ++ {CAT_AIME, "Q: How many positive integers less than 100 are divisible by 3 or 5?\nA:", "47"}, ++ {CAT_AIME, "Q: What is the sum of the first 10 positive even numbers?\nA:", "110"}, ++ {CAT_AIME, "Q: How many diagonals does a regular octagon have?\nA:", "20"}, ++ {CAT_AIME, "Q: What is the greatest common divisor of 84 and 126?\nA:", "42"}, ++ {CAT_AIME, "Q: In how many ways can the letters of the word LEVEL be arranged?\nA:", "30"}, ++ {CAT_AIME, "Q: What is the sum of the interior angles of a hexagon, in degrees?\nA:", "720"}, ++ // --- coding (execution traces / algorithmic output-prediction; harder, no exec) --- ++ {CAT_CODE, "Q: In Python, what does [x*x for x in range(4)] evaluate to?\nA:", "[0, 1, 4, 9]"}, ++ {CAT_CODE, "Q: In Python, what does list(zip([1, 2], [3, 4])) evaluate to?\nA:", "[(1, 3), (2, 4)]"}, ++ {CAT_CODE, "Q: In Python, what does int('1010', 2) evaluate to?\nA:", "10"}, ++ {CAT_CODE, "Q: In Python, what does 5 & 3 evaluate to?\nA:", "1"}, ++ {CAT_CODE, "Q: In Python, what does len({'a': 1, 'b': 2, 'a': 3}) return?\nA:", "2"}, ++ {CAT_CODE, "Q: In Python, what does 'abracadabra'.count('a') return?\nA:", "5"}, ++ // aliasing gotcha: b is a, so a mutates too ++ {CAT_CODE, "Q: What does this Python code print?\na = [1, 2, 3]\nb = a\nb.append(4)\nprint(len(a))\nA:", "4"}, ++ // accumulator trace ++ {CAT_CODE, "Q: What does this Python code print?\nx = 0\nfor i in range(5):\n x += i\nprint(x)\nA:", "10"}, ++ // dict counting trace ++ {CAT_CODE, "Q: What does this Python code print?\nd = {}\nfor c in 'aabbbc':\n d[c] = d.get(c, 0) + 1\nprint(d['b'])\nA:", "3"}, ++ // recursion: factorial ++ {CAT_CODE, "Q: A function is defined by f(0)=1 and f(n)=n*f(n-1). What is f(5)?\nA:", "120"}, ++ // fibonacci by definition ++ {CAT_CODE, "Q: The Fibonacci sequence starts 1, 1, 2, 3, 5. What is the 8th Fibonacci number?\nA:", "21"}, ++ // binary search / algorithmic result ++ {CAT_CODE, "Q: In Python, what does sorted([3, 1, 2], reverse=True) return?\nA:", "[3, 2, 1]"}, ++ // --- factual / knowledge (moderate; balances the numeric-heavy set) --- ++ {CAT_FACT, "Q: The chemical symbol for gold is\nA:", "Au"}, ++ {CAT_FACT, "Q: The powerhouse of the cell is the\nA:", "mitochondria"}, ++ {CAT_FACT, "Q: In what year did World War II end?\nA:", "1945"}, ++ {CAT_FACT, "Q: The square root of 169 is\nA:", "13"}, ++ {CAT_FACT, "Q: The largest planet in the solar system is\nA:", "Jupiter"}, ++}; ++ + struct Candidate { + int i = 0, j = 0; // duplicate source layers [i, j) + std::string spec; // "i,j" (empty = base) + double overhead_pct = 0.0; // (j-i)/n_transformer * 100 + bool ok = false; + double ppl = 0.0; +- double dppl = 0.0; // ppl - base_ppl (negative = better) +- double task = 0.0; // fraction correct on the task-probe +-}; +- +-struct TaskQA { const char * q; const char * a; }; +- +-// Built-in graded probe: arithmetic, multi-step word problems, factual recall, +-// and sequence completion (base-model friendly, greedy, substring-graded). ~24 +-// items so the `task` score has real resolution (1/24 ≈ 0.04) to RANK templates, +-// not just flag a broken one. Greedy decode early-stops as soon as the answer +-// substring appears, so cost is a few tokens/item in practice. +-const TaskQA kTasks[] = { +- {"Q: What is 17 + 26?\nA:", "43"}, +- {"Q: What is 8 times 7?\nA:", "56"}, +- {"Q: What is 100 minus 37?\nA:", "63"}, +- {"Q: What is 144 divided by 12?\nA:", "12"}, +- {"Q: What is 9 times 6?\nA:", "54"}, +- {"Q: What is 15 plus 27?\nA:", "42"}, +- {"Q: What is 81 divided by 9?\nA:", "9"}, +- {"Q: What is 13 times 4?\nA:", "52"}, +- {"Q: What is 250 minus 75?\nA:", "175"}, +- {"Q: What is half of 96?\nA:", "48"}, +- {"Q: What is 7 squared?\nA:", "49"}, +- {"Q: What is 3 to the power of 4?\nA:", "81"}, +- {"Q: If a train travels 60 km in 2 hours, its speed in km/h is\nA:", "30"}, +- {"Q: A shop sells pens at 3 dollars each. Five pens cost how many dollars?\nA:", "15"}, +- {"Q: Tom has 12 apples and gives away 5. How many are left?\nA:", "7"}, +- {"Q: There are 24 hours in a day. How many hours are in 3 days?\nA:", "72"}, +- {"Q: The capital of France is\nA:", "Paris"}, +- {"Q: The capital of Japan is\nA:", "Tokyo"}, +- {"Q: The chemical symbol for water is\nA:", "H2O"}, +- {"Q: The largest planet in the solar system is\nA:", "Jupiter"}, +- {"Q: The opposite of hot is\nA:", "cold"}, +- {"Q: Complete the sequence: 2, 4, 6, 8,\nA:", "10"}, +- {"Q: Complete the sequence: 1, 2, 4, 8,\nA:", "16"}, +- {"Q: How many sides does a triangle have?\nA:", "3"}, ++ double dppl = 0.0; // ppl - base_ppl (negative = better) — SECONDARY ++ double task = 0.0; // fraction correct on the battery — PRIMARY ++ double dtask = 0.0; // task - base_task (positive = capability gain) ++ double cat[N_CAT] = {0}; // per-category fraction correct + }; + + // A tiny default corpus used when the user passes no -f/-p (enough tokens for a +@@ -145,18 +188,52 @@ bool eval_ppl(llama_context * ctx, const llama_vocab * vocab, + return true; + } + +-// Greedy generate a short answer per task; score = fraction whose answer string +-// appears in the generation. +-double eval_task(llama_context * ctx, const llama_vocab * vocab, int n_gen) { ++// An answer of pure-integer form must match as a WHOLE number (digit boundaries ++// on both sides), so "3" doesn't spuriously match "30"/"137" in a rambling ++// greedy trace — the biggest source of grading noise for short numeric answers. ++bool is_pure_int(const std::string & s) { ++ if (s.empty()) return false; ++ size_t k = (s[0] == '-') ? 1 : 0; ++ if (k >= s.size()) return false; ++ for (; k < s.size(); ++k) if (!std::isdigit((unsigned char) s[k])) return false; ++ return true; ++} ++ ++bool contains_number(const std::string & out, const std::string & ans) { ++ size_t pos = 0; ++ while ((pos = out.find(ans, pos)) != std::string::npos) { ++ const bool lok = (pos == 0) || !std::isdigit((unsigned char) out[pos - 1]); ++ const size_t end = pos + ans.size(); ++ const bool rok = (end >= out.size()) || !std::isdigit((unsigned char) out[end]); ++ if (lok && rok) return true; ++ pos = end; ++ } ++ return false; ++} ++ ++// Numeric answers → whole-number boundary match; everything else (lists, ++// fractions, words, code) → plain substring. Both are order-insensitive so a ++// short greedy chain-of-thought that lands the answer still scores. ++bool answer_hit(const std::string & out, const std::string & ans) { ++ if (is_pure_int(ans)) return contains_number(out, ans); ++ return out.find(ans) != std::string::npos; ++} ++ ++// Greedy-generate a short answer per task; accumulate per-category correct/total. ++// Returns the overall fraction correct and fills out_cat[N_CAT] with per-category ++// fractions. Generation early-stops on answer-hit, on the model rolling into a ++// fake next "Q" item, or on EOS. ++double eval_task(llama_context * ctx, const llama_vocab * vocab, double out_cat[N_CAT]) { + const int n_vocab = llama_vocab_n_tokens(vocab); + const llama_token eos = llama_vocab_eos(vocab); +- int correct = 0, total = 0; ++ int correct[N_CAT] = {0}, total[N_CAT] = {0}; + llama_batch batch = llama_batch_init(512, 0, 1); + for (const TaskQA & t : kTasks) { ++ ++total[t.cat]; + llama_memory_clear(llama_get_memory(ctx), true); + std::vector p = common_tokenize(vocab, t.q, true, false); + const int np = (int) p.size(); +- if (np < 1 || np > 512) { ++total; continue; } ++ if (np < 1 || np > 512) continue; + batch.n_tokens = np; + for (int k = 0; k < np; ++k) { + batch.token[k] = p[k]; +@@ -165,15 +242,17 @@ double eval_task(llama_context * ctx, const llama_vocab * vocab, int n_gen) { + batch.seq_id[k][0] = 0; + batch.logits[k] = (k == np - 1) ? 1 : 0; + } +- if (llama_decode(ctx, batch)) { ++total; continue; } ++ if (llama_decode(ctx, batch)) continue; + std::string out; + int n_past = np; + bool hit = false; ++ const int n_gen = kCatGen[t.cat]; + llama_token tok = argmax_row(llama_get_logits_ith(ctx, -1), n_vocab); + for (int g = 0; g < n_gen; ++g) { + if (tok == eos) break; + out += common_token_to_piece(ctx, tok, false); +- if (out.find(t.a) != std::string::npos) { hit = true; break; } // early-stop ++ if (answer_hit(out, t.a)) { hit = true; break; } // early-stop on hit ++ if (out.find("\nQ") != std::string::npos) break; // rolled to next item + batch.n_tokens = 1; + batch.token[0] = tok; + batch.pos[0] = n_past; +@@ -184,16 +263,21 @@ double eval_task(llama_context * ctx, const llama_vocab * vocab, int n_gen) { + ++n_past; + tok = argmax_row(llama_get_logits_ith(ctx, -1), n_vocab); + } +- ++total; +- if (hit) ++correct; ++ if (hit) ++correct[t.cat]; + } + llama_batch_free(batch); +- return total ? (double) correct / (double) total : 0.0; ++ int C = 0, T = 0; ++ for (int c = 0; c < N_CAT; ++c) { ++ out_cat[c] = total[c] ? (double) correct[c] / (double) total[c] : 0.0; ++ C += correct[c]; ++ T += total[c]; ++ } ++ return T ? (double) C / (double) T : 0.0; + } + + // Build a context for this candidate plan, score it, free it. + bool score_candidate(llama_model * model, const common_params & base, +- const std::vector & corpus_toks, int n_gen, ++ const std::vector & corpus_toks, + Candidate & c) { + common_params p = base; // copy; `spec` must outlive the ctor call + const std::string spec = c.spec; +@@ -203,9 +287,9 @@ bool score_candidate(llama_model * model, const common_params & base, + if (!ctx) return false; // arch-unsupported / boundary reject / OOM + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool ppl_ok = eval_ppl(ctx, vocab, corpus_toks, p.n_ctx, p.n_batch, c.ppl); +- c.task = eval_task(ctx, vocab, n_gen); ++ c.task = eval_task(ctx, vocab, c.cat); // PRIMARY signal; fills c.cat[] + llama_free(ctx); +- c.ok = ppl_ok; ++ c.ok = ppl_ok; // ppl_ok = the context ran; task always scored + return ppl_ok; + } + +@@ -312,82 +396,115 @@ int rys_probe_main(int argc, char ** argv) { + + // Base (empty plan) reference first. + Candidate base; base.spec = ""; base.overhead_pct = 0.0; +- if (!score_candidate(model, params, corpus_toks, 24, base)) { +- fprintf(stderr, "rys-probe: base PPL failed — cannot score (bad corpus or model?)\n"); ++ if (!score_candidate(model, params, corpus_toks, base)) { ++ fprintf(stderr, "rys-probe: base scoring failed — cannot proceed (bad corpus or model?)\n"); + return 1; + } +- fprintf(stderr, "rys-probe: base ppl=%.4f task=%.3f\n", base.ppl, base.task); ++ fprintf(stderr, "rys-probe: base task=%.3f (", base.task); ++ for (int c = 0; c < N_CAT; ++c) ++ fprintf(stderr, "%s%s=%.2f", c ? " " : "", kCatName[c], base.cat[c]); ++ fprintf(stderr, ") ppl=%.4f\n", base.ppl); ++ if (base.task > 0.95) ++ fprintf(stderr, "rys-probe: WARNING base task=%.3f is near-saturated — little " ++ "headroom to detect a gain; consider a harder battery.\n", base.task); + + // Score each candidate. + size_t done = 0; + for (Candidate & c : cands) { +- score_candidate(model, params, corpus_toks, 24, c); +- c.dppl = c.ok ? (c.ppl - base.ppl) : 0.0; ++ score_candidate(model, params, corpus_toks, c); ++ c.dppl = c.ok ? (c.ppl - base.ppl) : 0.0; ++ c.dtask = c.ok ? (c.task - base.task) : 0.0; + ++done; +- fprintf(stderr, "\rrys-probe: scored %zu/%zu (last %s: %s) ", +- done, cands.size(), c.spec.c_str(), c.ok ? "ok" : "FAIL"); ++ fprintf(stderr, "\rrys-probe: scored %zu/%zu (last %s: task%+.3f) ", ++ done, cands.size(), c.spec.c_str(), c.ok ? c.dtask : 0.0); + fflush(stderr); + } + fprintf(stderr, "\n"); + +- // Valid = beats base PPL AND does not regress the task-probe (guardrail so a +- // boundary/broken block that happens to lower PPL can't win). ++ // WIN = strictly raises the task battery over base (dtask > 0). PPL is NOT a ++ // gate — a duplication only earns its tps cost if it improves CAPABILITY. + std::vector wins; + for (Candidate & c : cands) +- if (c.ok && c.dppl < 0.0 && c.task >= base.task) wins.push_back(&c); ++ if (c.ok && c.dtask > 0.0) wins.push_back(&c); ++ ++ // Rank by Δtask DESC (primary); ties broken by lower ppl, then fewer layers. ++ auto by_dtask = [](const Candidate * a, const Candidate * b) { ++ if (a->dtask != b->dtask) return a->dtask > b->dtask; ++ if (a->ppl != b->ppl) return a->ppl < b->ppl; ++ return (a->j - a->i) < (b->j - b->i); ++ }; ++ std::sort(wins.begin(), wins.end(), by_dtask); + +- // Context table: top-N wins by PPL gain (sorted ascending PPL). +- std::sort(wins.begin(), wins.end(), +- [](const Candidate * a, const Candidate * b) { return a->ppl < b->ppl; }); + const int topk = params.rys_probe_topk > 0 ? params.rys_probe_topk : 10; +- printf("\n=== RYS probe (base ppl=%.4f, task=%.3f) — top wins by PPL ===\n", base.ppl, base.task); +- printf("%-12s %5s %8s %10s %10s %8s\n", "block", "+lyr", "over%", "ppl", "dppl", "task"); +- for (size_t k = 0; k < wins.size() && (int) k < topk; ++k) { +- const Candidate * c = wins[k]; +- printf("%-12s %5d %7.2f%% %10.4f %+10.4f %8.3f\n", +- c->spec.c_str(), c->j - c->i, c->overhead_pct, c->ppl, c->dppl, c->task); ++ printf("\n=== RYS probe (base task=%.3f, ppl=%.4f) — top blocks by Δtask ===\n", ++ base.task, base.ppl); ++ printf("%-12s %5s %8s %8s %9s %10s %9s\n", ++ "block", "+lyr", "over%", "task", "dtask", "ppl", "dppl"); ++ // If nothing wins, still show the best-effort top-K (by dtask incl. ≤0) so the ++ // user sees how close/far it is, and which category (if any) moved. ++ std::vector shown = wins; ++ if (shown.empty()) { ++ for (Candidate & c : cands) if (c.ok) shown.push_back(&c); ++ std::sort(shown.begin(), shown.end(), by_dtask); ++ } ++ for (size_t k = 0; k < shown.size() && (int) k < topk; ++k) { ++ const Candidate * c = shown[k]; ++ printf("%-12s %5d %7.2f%% %8.3f %+9.3f %10.4f %+9.4f\n", ++ c->spec.c_str(), c->j - c->i, c->overhead_pct, c->task, c->dtask, ++ c->ppl, c->dppl); + } + + if (wins.empty()) { +- printf("\nNo candidate beat base PPL without regressing the task-probe. " +- "Try a larger/domain corpus (-f) or a different --rys-probe-band.\n"); +- printf("\nNOTE: PPL+task shortlist, not a verdict — validate on your real harness " ++ printf("\nNO block raised the task battery over base (best Δtask %+.3f). On this " ++ "model + battery, RYS layer duplication does NOT buy capability — and it " ++ "COSTS decode tps (extra effective layers + KV). Recommendation: do not " ++ "duplicate.\n", shown.empty() ? 0.0 : shown.front()->dtask); ++ printf(" Base per-category: "); ++ for (int c = 0; c < N_CAT; ++c) printf("%s=%.2f ", kCatName[c], base.cat[c]); ++ printf("\n (If a category is at 0.00 or 1.00 it has no headroom — pass -f a " ++ "domain corpus or widen the battery to probe a skill that can move.)\n"); ++ printf("\nNOTE: shortlist, not a verdict. Δtask is greedy-graded on a small " ++ "battery — confirm any candidate on your real harness " + "(perf/llamafile/rys-probe-eval.sh; see docs/features/rys_probe.md).\n"); + return 0; + } + + // The two templates the user picks between: +- // MAX GAIN = lowest PPL (task already >= base). +- // MOST EFFICIENT = largest PPL drop PER ADDED LAYER (best bang per compute). +- Candidate * gain = wins.front(); // sorted ascending by ppl +- auto eff_of = [&](const Candidate * c) { +- return (base.ppl - c->ppl) / (double) (c->j - c->i); ++ // MAX GAIN = highest Δtask (best measured capability). ++ // MOST EFFICIENT = highest Δtask PER ADDED LAYER (best capability per tps cost). ++ Candidate * gain = wins.front(); // sorted: highest dtask first ++ auto eff_of = [](const Candidate * c) { ++ return c->dtask / (double) (c->j - c->i); + }; + Candidate * eff = wins.front(); + for (Candidate * c : wins) if (eff_of(c) > eff_of(eff)) eff = c; + +- // Ready-to-paste bullet lines. "params" = extra layer weight re-run (RYS is +- // weight-shared, so this is compute overhead, not extra param VRAM). ++ // "params" = extra layer weight re-run (RYS is weight-shared, so this is ++ // compute/KV overhead, not extra param VRAM). + const int eff_n = eff->j - eff->i; + const int gain_n = gain->j - gain->i; + const double eff_p = (double) ((long) (eff->overhead_pct * 10 + 0.5)) / 10.0; // 1-dp, trimmed + const double gain_p = (double) ((long) (gain->overhead_pct * 10 + 0.5)) / 10.0; + +- printf("\n=== 2 templates ===\n"); +- printf("- MOST EFFICIENT → --repeat-layers %s — +%d %s, %g%% params, ppl %.2f→%.2f, task %.3f.\n", ++ printf("\n=== 2 templates (ranked by Δtask; PPL secondary) ===\n"); ++ printf("- MOST EFFICIENT → --repeat-layers %s — +%d %s, %g%% overhead, task %.3f→%.3f " ++ "(%+.3f, %+.4f/layer), ppl %.2f→%.2f.\n", + eff->spec.c_str(), eff_n, eff_n == 1 ? "layer" : "layers", eff_p, +- base.ppl, eff->ppl, eff->task); +- printf("- MAX GAIN → --repeat-layers %s — +%d %s, %g%% params, ppl %.2f→%.2f, task %.3f.\n", ++ base.task, eff->task, eff->dtask, eff_of(eff), base.ppl, eff->ppl); ++ printf("- MAX GAIN → --repeat-layers %s — +%d %s, %g%% overhead, task %.3f→%.3f " ++ "(%+.3f), ppl %.2f→%.2f.\n", + gain->spec.c_str(), gain_n, gain_n == 1 ? "layer" : "layers", gain_p, +- base.ppl, gain->ppl, gain->task); ++ base.task, gain->task, gain->dtask, base.ppl, gain->ppl); + if (eff == gain) +- printf(" (the same block is both — no wider block gained more.)\n"); +- printf(" (detail: MOST EFFICIENT %.4f ppl/layer, dppl %+.4f; MAX GAIN dppl %+.4f. base ppl %.4f task %.3f.)\n", +- eff_of(eff), eff->dppl, gain->dppl, base.ppl, base.task); +- +- printf("\nNOTE: PPL+task shortlist, not a verdict. Validate BOTH templates on a real task " +- "eval with perf/llamafile/rys-probe-eval.sh (boots the server per spec + base and " +- "runs your external harness). See docs/features/rys_probe.md.\n"); ++ printf(" (the same block is both — no other block gained more.)\n"); ++ printf(" MAX GAIN per-category Δ: "); ++ for (int c = 0; c < N_CAT; ++c) printf("%s%+.2f ", kCatName[c], gain->cat[c] - base.cat[c]); ++ printf("\n"); ++ ++ printf("\nNOTE: Δtask-ranked shortlist, not a verdict. The battery is small and " ++ "greedy-graded — validate BOTH templates on a real task eval with " ++ "perf/llamafile/rys-probe-eval.sh (boots the server per spec + base and runs " ++ "your external harness). See docs/features/rys_probe.md.\n"); + + return 0; + } diff --git a/patches/0132-runtime-introspection.patch b/patches/0132-runtime-introspection.patch new file mode 100644 index 0000000000000000000000000000000000000000..a75d01bb025d405e86a3ab1b42158959252f94e9 --- /dev/null +++ b/patches/0132-runtime-introspection.patch @@ -0,0 +1,384 @@ +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index 20141023c..a67024315 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -869,6 +869,24 @@ extern "C" { + // Check if the memory supports shifting + LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + ++ // opencoti #676 runtime introspection — effective KV-cache state after boot-time policy ++ // (auto KV-tier, rolling-KV residency sizing). Reports the ATTENTION KV cache: an iSWA ++ // memory reports its non-SWA base cache (is_iswa = true), a hybrid memory reports its ++ // attention child. Returns false when the memory has no KV cache (pure recurrent) or ++ // info is NULL. Boot-static: safe to read once after context creation and cache. ++ struct llama_opencoti_kv_info { ++ enum ggml_type type_k; // effective K type of the resident window (post auto-tier) ++ enum ggml_type type_v; // effective V type of the resident window (post auto-tier) ++ enum ggml_type type_k_tail; // spilled-tail K type (== type_k when no distinct tail) ++ enum ggml_type type_v_tail; // spilled-tail V type (== type_v when no distinct tail) ++ uint32_t n_cells; // allocated KV cells per stream ++ uint32_t n_cells_resident; // device-resident window cells (min across spilling layers); ++ // == n_cells when fully resident ++ uint32_t n_layers_spilling; // KV layers with an engaged rolling-KV position window ++ bool is_iswa; // reported from an iSWA base (full-attention) cache ++ }; ++ LLAMA_API bool llama_memory_opencoti_kv_info(llama_memory_t mem, struct llama_opencoti_kv_info * info); ++ + // + // State / sessions + // +diff --git a/llama.cpp/src/llama-memory.h b/llama.cpp/src/llama-memory.h +index 98e4f269b..12cb65512 100644 +--- a/llama.cpp/src/llama-memory.h ++++ b/llama.cpp/src/llama-memory.h +@@ -140,6 +140,14 @@ struct llama_memory_i { + return {}; + } + ++ // opencoti #676 runtime introspection — see llama.h llama_opencoti_kv_info. The default ++ // reports nothing (pure recurrent memories have no KV cache), keeping every non-KV memory ++ // byte-for-byte unaffected. ++ virtual bool opencoti_kv_info(struct llama_opencoti_kv_info & info) const { ++ (void) info; ++ return false; ++ } ++ + virtual std::map memory_breakdown() const = 0; + + // +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +index 7045ec55e..d78ef7e31 100644 +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -270,6 +270,9 @@ public: + ggml_type type_k_tail() const; + ggml_type type_v_tail() const; + ++ // opencoti #676 runtime introspection — effective types + residency summary. ++ bool opencoti_kv_info(struct llama_opencoti_kv_info & info) const override; ++ + // + // graph_build API + // +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index eb5899f45..f8c6aab01 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -2915,6 +2915,29 @@ ggml_type llama_kv_cache::type_v_tail() const { + return type_v(); + } + ++// opencoti #676 runtime introspection — effective post-boot-policy KV state (auto-tier result, ++// rolling-KV residency summary). Read-only over boot-static fields; see llama.h. ++bool llama_kv_cache::opencoti_kv_info(struct llama_opencoti_kv_info & info) const { ++ if (layers.empty()) { ++ return false; ++ } ++ info.type_k = type_k(); ++ info.type_v = type_v(); ++ info.type_k_tail = type_k_tail(); ++ info.type_v_tail = type_v_tail(); ++ info.n_cells = get_size(); ++ info.n_cells_resident = info.n_cells; ++ info.n_layers_spilling = 0; ++ for (const auto & layer : layers) { ++ if (layer.window_cells > 0) { ++ info.n_layers_spilling++; ++ info.n_cells_resident = std::min(info.n_cells_resident, layer.window_cells); ++ } ++ } ++ info.is_iswa = false; ++ return true; ++} ++ + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { + uint32_t result = 0; + +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 5018928f4..1e8824248 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -121,6 +121,9 @@ public: + llama_kv_cache * get_base() const; + llama_kv_cache * get_swa () const; + ++ // opencoti #676 runtime introspection — forwards to the non-SWA base cache. ++ bool opencoti_kv_info(struct llama_opencoti_kv_info & info) const override; ++ + private: + const llama_hparams & hparams; + +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index 584370965..81de1a3f4 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -292,6 +292,16 @@ llama_kv_cache * llama_kv_cache_iswa::get_swa() const { + return kv_swa.get(); + } + ++// opencoti #676 runtime introspection — report the non-SWA base (full-attention) cache; the SWA ++// cache shares its types and is bounded by the window, so the base is the informative one. ++bool llama_kv_cache_iswa::opencoti_kv_info(struct llama_opencoti_kv_info & info) const { ++ if (!kv_base || !kv_base->opencoti_kv_info(info)) { ++ return false; ++ } ++ info.is_iswa = true; ++ return true; ++} ++ + // + // llama_kv_cache_iswa_context + // +diff --git a/llama.cpp/src/llama-memory-hybrid.h b/llama.cpp/src/llama-memory-hybrid.h +index e41ba4fed..353893098 100644 +--- a/llama.cpp/src/llama-memory-hybrid.h ++++ b/llama.cpp/src/llama-memory-hybrid.h +@@ -99,6 +99,9 @@ public: + llama_kv_cache * get_mem_attn() const; + llama_memory_recurrent * get_mem_recr() const; + ++ // opencoti #676 runtime introspection — forwards to the attention child cache. ++ bool opencoti_kv_info(struct llama_opencoti_kv_info & info) const override; ++ + private: + const llama_hparams & hparams; + +diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp +index 67ecee16c..16ec5380d 100644 +--- a/llama.cpp/src/llama-memory-hybrid.cpp ++++ b/llama.cpp/src/llama-memory-hybrid.cpp +@@ -258,6 +258,11 @@ llama_memory_recurrent * llama_memory_hybrid::get_mem_recr() const { + return mem_recr.get(); + } + ++// opencoti #676 runtime introspection — the attention child carries the KV cache. ++bool llama_memory_hybrid::opencoti_kv_info(struct llama_opencoti_kv_info & info) const { ++ return mem_attn && mem_attn->opencoti_kv_info(info); ++} ++ + llama_memory_hybrid_context::llama_memory_hybrid_context(llama_memory_status status) : status(status) {} + + llama_memory_hybrid_context::llama_memory_hybrid_context(llama_memory_hybrid * mem) : +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 20423d5a0..4f9f27a52 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -4444,6 +4444,15 @@ bool llama_memory_can_shift(llama_memory_t mem) { + return mem->get_can_shift(); + } + ++// opencoti #676 runtime introspection ++bool llama_memory_opencoti_kv_info(llama_memory_t mem, struct llama_opencoti_kv_info * info) { ++ if (!mem || !info) { ++ return false; ++ } ++ ++ return mem->opencoti_kv_info(*info); ++} ++ + // llama state API + + // deprecated +diff --git a/llama.cpp/tools/server/server-context.h b/llama.cpp/tools/server/server-context.h +index 65853438c..fb41793cd 100644 +--- a/llama.cpp/tools/server/server-context.h ++++ b/llama.cpp/tools/server/server-context.h +@@ -50,6 +50,11 @@ struct server_context_meta { + int32_t model_n_embd_inp; + uint64_t model_n_params; + uint64_t model_size; ++ ++ // opencoti #676 — boot-state echo of the opencoti feature set, served under /props ++ // "opencoti". Computed once at get_meta() because /props must stay servable while ++ // sleeping (it must not dereference the live context). ++ json json_opencoti; + }; + + struct server_context { +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 39752352f..97d7011c7 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -180,6 +180,11 @@ struct server_slot { + int32_t n_draft_total = 0; // Total draft tokens generated + int32_t n_draft_accepted = 0; // Draft tokens actually accepted + ++ // opencoti #677 — lifetime accumulators for /slots introspection; unlike the ++ // per-task counters above these are NOT cleared in reset() ++ int32_t n_draft_total_cum = 0; ++ int32_t n_draft_accepted_cum = 0; ++ + void reset() { + SLT_DBG(*this, "%s", "\n"); + +@@ -526,6 +531,23 @@ struct server_slot { + } + } + ++ // opencoti #677 — live agentic introspection: cumulative draft acceptance for this ++ // slot plus the session / SharedKVPool binding of the current (or last) task. ++ { ++ json oc = { ++ { "draft_n_total", n_draft_total_cum }, ++ { "draft_n_accepted", n_draft_accepted_cum }, ++ { "draft_acceptance", n_draft_total_cum > 0 ++ ? json((double) n_draft_accepted_cum / (double) n_draft_total_cum) : json() }, ++ }; ++ if (ptask) { ++ oc["session_id"] = ptask->params.session_id; ++ oc["shared_pool_slot"] = ptask->params.shared_pool_slot; ++ oc["shared_prefix_n_tokens"] = ptask->params.shared_prefix_n_tokens; ++ } ++ res["opencoti"] = oc; ++ } ++ + return res; + } + +@@ -2708,7 +2730,8 @@ private: + auto & draft = slot.spec_draft; + auto & ckpt = slot.spec_ckpt; + +- slot.n_draft_total += draft.size(); ++ slot.n_draft_total += draft.size(); ++ slot.n_draft_total_cum += draft.size(); // opencoti #677 + + // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] + const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; +@@ -3695,7 +3718,8 @@ private: + slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; + + // update how many tokens out of those tested were accepted +- slot.n_draft_accepted += ids.size() - 1; ++ slot.n_draft_accepted += ids.size() - 1; ++ slot.n_draft_accepted_cum += ids.size() - 1; // opencoti #677 + + // add accepted tokens to the prompt + slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft); +@@ -3743,6 +3767,93 @@ private: + return slots.back().n_ctx; + } + ++ // opencoti #676 runtime introspection — boot-state of the opencoti feature set, exposed ++ // verbatim under /props "opencoti". "configured" mirrors the CLI/env knobs; "kv.effective" ++ // is read back from the live KV cache so it reflects boot policy (auto KV-tier, rolling-KV ++ // residency sizing) rather than what was asked for. ++ json get_opencoti_props() { ++ const auto & p = params_base; ++ ++ auto tail_or_null = [](ggml_type t) -> json { ++ return t == GGML_TYPE_COUNT ? json() : json(ggml_type_name(t)); ++ }; ++ ++ json kv = { ++ { "cache_type_k", ggml_type_name(p.cache_type_k) }, ++ { "cache_type_v", ggml_type_name(p.cache_type_v) }, ++ { "cache_type_k_tail", tail_or_null(p.cache_type_k_tail) }, ++ { "cache_type_v_tail", tail_or_null(p.cache_type_v_tail) }, ++ { "auto_tier", getenv("OPENCOTI_KV_AUTO_TIER") != nullptr }, ++ { "auto_tier_tail", getenv("OPENCOTI_KV_AUTO_TIER_TAIL") != nullptr }, ++ }; ++ llama_opencoti_kv_info ki; ++ if (ctx_tgt && llama_memory_opencoti_kv_info(llama_get_memory(ctx_tgt), &ki)) { ++ kv["effective"] = json { ++ { "type_k", ggml_type_name(ki.type_k) }, ++ { "type_v", ggml_type_name(ki.type_v) }, ++ { "type_k_tail", ggml_type_name(ki.type_k_tail) }, ++ { "type_v_tail", ggml_type_name(ki.type_v_tail) }, ++ { "n_cells", ki.n_cells }, ++ { "n_cells_resident", ki.n_cells_resident }, ++ { "n_layers_spilling", ki.n_layers_spilling }, ++ { "fully_resident", ki.n_layers_spilling == 0 }, ++ { "is_iswa", ki.is_iswa }, ++ }; ++ } ++ ++ json spec_types = json::array(); ++ for (const auto t : p.speculative.types) { ++ spec_types.push_back(common_speculative_type_to_str(t)); ++ } ++ ++ return json { ++ { "kv", kv }, ++ { "residency", json { ++ { "kv_residency_mode", p.kv_residency_mode }, ++ { "vram_target_mib", p.vram_target_mib }, ++ { "headinfer_gpu_heads_frac", p.headinfer_gpu_heads_frac }, ++ } }, ++ { "dca", json { ++ { "enabled", p.dca_enabled }, ++ { "chunk_size", p.dca_chunk_size }, ++ { "yarn_factor", p.dca_yarn_factor }, ++ } }, ++ { "sparse_attn", json { ++ { "enabled", p.sparse_attn_enabled }, ++ { "block_size", p.sparse_attn_block_size }, ++ { "topk", p.sparse_attn_topk }, ++ { "recent", p.sparse_attn_recent }, ++ { "sink", p.sparse_attn_sink }, ++ { "refresh", p.sparse_attn_refresh }, ++ { "mode", p.sparse_attn_mode }, ++ } }, ++ { "speculative", json { ++ { "types", spec_types }, ++ { "draft_model", p.speculative.draft.mparams.path }, ++ { "n_max", p.speculative.draft.n_max }, ++ { "n_min", p.speculative.draft.n_min }, ++ { "draft_block_size", p.speculative.draft.draft_block_size }, ++ { "n_gpu_layers", p.speculative.draft.n_gpu_layers }, ++ { "cache_type_k", ggml_type_name(p.speculative.draft.cache_type_k) }, ++ { "cache_type_v", ggml_type_name(p.speculative.draft.cache_type_v) }, ++ } }, ++ { "kv_reuse", json { ++ { "n_parallel", p.n_parallel }, ++ { "kv_unified", p.kv_unified }, ++ { "cache_reuse", p.n_cache_reuse }, ++ { "slot_prompt_similarity", p.slot_prompt_similarity }, ++ { "ctx_checkpoints", p.n_ctx_checkpoints }, ++ { "cache_ram_mib", p.cache_ram_mib }, ++ } }, ++ { "rest_kv", json { ++ { "eviction", p.rest_kv_eviction }, ++ { "recent", p.rest_kv_recent }, ++ { "layer", p.rest_kv_layer }, ++ } }, ++ { "repeat_layers", p.repeat_layers.empty() ? json() : json(p.repeat_layers) }, ++ }; ++ } ++ + server_response_reader get_response_reader() { + return server_response_reader(queue_tasks, queue_results, HTTP_POLLING_SECONDS); + } +@@ -3816,6 +3927,8 @@ server_context_meta server_context::get_meta() const { + /* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt), + /* model_n_params */ llama_model_n_params(impl->model_tgt), + /* model_size */ llama_model_size(impl->model_tgt), ++ ++ /* json_opencoti */ impl->get_opencoti_props(), // opencoti #676 + }; + } + +@@ -4386,6 +4499,7 @@ void server_routes::init_routes() { + { "build_info", meta->build_info }, + { "is_sleeping", queue_tasks.is_sleeping() }, + { "cors_proxy_enabled", params.ui_mcp_proxy || params.webui_mcp_proxy }, ++ { "opencoti", meta->json_opencoti }, // opencoti #676 runtime introspection + }; + if (params.use_jinja) { + if (!tmpl_tools.empty()) { +diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk +index 49c8979..9ec3823 100644 +--- a/llamafile/BUILD.mk ++++ b/llamafile/BUILD.mk +@@ -298,7 +298,10 @@ LLAMAFILE_SERVER_INCS := \ + -iquote o/$(MODE)/llama.cpp/tools/server + + # Compile server.cpp +-o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp ++# opencoti: depend on the server headers too — this rule has no .d tracking, and a ++# stale object here links an old inline update_meta with a mismatched ++# server_context_meta layout (silent ABI skew, bug-2186) ++o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp $(wildcard llama.cpp/tools/server/*.h) + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) $(LLAMAFILE_SERVER_INCS) -DLLAMA_BUILD_WEBUI -c -o $@ $< + diff --git a/patches/0133-opencoti-version-string.patch b/patches/0133-opencoti-version-string.patch new file mode 100644 index 0000000000000000000000000000000000000000..fedb441a39d6ea8776b1aad1c81aa50f59d0f139 --- /dev/null +++ b/patches/0133-opencoti-version-string.patch @@ -0,0 +1,67 @@ +diff --git a/llamafile/version.h b/llamafile/version.h +index aeb76619c..c0a7b9dd1 100644 +--- a/llamafile/version.h ++++ b/llamafile/version.h +@@ -27,3 +27,13 @@ + #define MKVERSION__(x, y, z) #x "." #y "." #z + #define MKVERSION_(x, y, z) MKVERSION__(x, y, z) + #define LLAMAFILE_VERSION_STRING MKVERSION_(LLAMAFILE_MAJOR, LLAMAFILE_MINOR, LLAMAFILE_PATCH) ++ ++// opencoti #613 — engine release tag. The single source of truth is ++// vendors/llamafile/OPENCOTI_TAG in the opencoti repo; the build pipeline ++// injects it as a make variable (-DOPENCOTI_TAG via llamafile/BUILD.mk). This ++// default only covers a bare `make` outside the pipeline — bump it together ++// with the tag file on every packaged cut. ++#ifndef OPENCOTI_TAG ++#define OPENCOTI_TAG "c3" ++#endif ++#define OPENCOTI_VERSION_STRING "opencoti-" LLAMAFILE_VERSION_STRING "-" OPENCOTI_TAG +diff --git a/llamafile/main.cpp b/llamafile/main.cpp +index 2707ddea9..c47e1685a 100644 +--- a/llamafile/main.cpp ++++ b/llamafile/main.cpp +@@ -107,7 +107,7 @@ int rys_probe_main(int argc, char **argv); + } // namespace lf + + static void print_general_help() { +- printf("llamafile v" LLAMAFILE_VERSION_STRING " - run LLMs locally\n" ++ printf("llamafile v" LLAMAFILE_VERSION_STRING " (" OPENCOTI_VERSION_STRING ") - run LLMs locally\n" + "\n" + "usage: llamafile -m MODEL.gguf [options]\n" + "\n" +@@ -283,7 +283,9 @@ int main(int argc, char **argv) { + + // Handle --version before anything else (ignores all other arguments) + if (llamafile_has(argv, "--version")) { +- puts("llamafile v" LLAMAFILE_VERSION_STRING); ++ // opencoti #613: full engine version string; "llamafile vX.Y.Z" stays first ++ // because version-parsing consumers grab the first x.y.z token ++ puts("llamafile v" LLAMAFILE_VERSION_STRING " (" OPENCOTI_VERSION_STRING ")"); + return 0; + } + +diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk +index 49c8979..9ec3823 100644 +--- a/llamafile/BUILD.mk ++++ b/llamafile/BUILD.mk +@@ -306,10 +309,18 @@ o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp + # Main executable + # ============================================================================== + ++# opencoti #613: pass the engine release tag through to version.h when the build ++# pipeline provides it (make OPENCOTI_TAG=cN, sourced from vendors/llamafile/ ++# OPENCOTI_TAG); the header carries a fallback default for bare `make`. ++ifneq ($(OPENCOTI_TAG),) ++OPENCOTI_TAG_FLAG := -DOPENCOTI_TAG=\"$(OPENCOTI_TAG)\" ++endif ++ + # main.cpp: no special includes needed (combined mode uses server_main via forward decl) +-o/$(MODE)/llamafile/main.o: llamafile/main.cpp ++# opencoti: explicit version.h dep — this rule has no .d tracking (bug-2183 class) ++o/$(MODE)/llamafile/main.o: llamafile/main.cpp llamafile/version.h + @mkdir -p $(@D) +- $(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) -c -o $@ $< ++ $(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) $(OPENCOTI_TAG_FLAG) -c -o $@ $< + + o/$(MODE)/llamafile/llamafile: \ + o/$(MODE)/llamafile/main.o \