diff --git "a/patches/0096-remove-single-ctx-gemma-mtp.patch" "b/patches/0096-remove-single-ctx-gemma-mtp.patch" new file mode 100644--- /dev/null +++ "b/patches/0096-remove-single-ctx-gemma-mtp.patch" @@ -0,0 +1,2308 @@ +From: opencoti +Subject: [PATCH 0096] remove single-context Gemma MTP facade; dual-ctx is the only assistant path (#611) + +END of the bug-858 port: delete the retired single-context in-target assistant +engine (~1600 lines) now that the dual-context port (0093) proved accept parity: +- llama-context: decode_mtp facade + decode_mtp_sync/fused, async worker + (decode_mtp_async/wait/run, mtp_worker thread, drain guard), process_ubatch_mtp, + graph_params_mtp, ensure_sched_mtp, sched_mtp/gf_res_prev_mtp, graph_compute_mtp. +- public API llama_decode_mtp removed (llama_decode_mtp_fused_nextn stays). +- llama-graph: build_attn_mtp (iswa cross-read); kv-cache-iswa::init_mtp. +- models: llm_build_gemma4_mtp + one-step builder + gemma4.cpp MTP dispatch; + gemma4-assistant keeps ONLY the dual-ctx graph (ctx_other required). +- speculative: common_speculative_impl_draft_assistant + NDJSON tracer removed; + --spec-type draft-assistant now requires ctx_dft (server always creates it). +- server: OPENCOTI_MTP_DUAL_CTX env gate removed — dual-ctx unconditional; + the --parallel>1 -> --kv-unified boot guard is preserved (shared-KV aliasing + has the same per-stream-split constraint the facade had). +NextN side untouched (mtp_fused_steps, sched_nextn, mtp_slot_info, unified twin +builders); mtp_assistant model loading kept. Gate on bs2: A4B n2 305.4 tps / +0.849 accept (== 0093 reference, boot line proves dual-ctx default), q35 n3 +248.8 / 0.662. bug-2110 (par2 accept 0.635 on the tps harness) was RESOLVED as +a harness artifact: on the real 9-prompt mtp-bench the dual-ctx assistant holds +accept at --parallel 2 --kv-unified (ours 0.785 par1 / 0.783 par2; upstream +b9859 0.793 / 0.792). + +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +index 3bb6aa2..887dfb6 100644 +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -305,7 +305,7 @@ struct common_params_speculative_draft { + + // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): OPTIONAL explicit draft-depth ceiling. + // 0 (default) = draft depth follows --spec-draft-n-max (n_max); the MTP head re-runs +- // autoregressively per step (decode_mtp_sync/fused), so depth is NOT limited to a fixed ++ // autoregressively per step (draft_mtp / decode_mtp_fused_nextn), so depth is NOT limited to a fixed + // block size. bug-858: the old default 3 hard-capped ours at 2 drafts/round and blocked + // n-max scaling vs upstream b9859. When set >1, caps draft depth at draft_block_size-1. + int32_t draft_block_size = 0; +diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp +index ee7962f..010bf65 100644 +--- a/llama.cpp/common/speculative.cpp ++++ b/llama.cpp/common/speculative.cpp +@@ -1409,315 +1409,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { + } + }; + +-// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. +-// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, +-// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV +-// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens +-// of a block in ONE facade call (no per-step sampler loop). +-// +-// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), +-// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never +-// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() +-// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) +-// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). +- +-// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. +-// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. +-namespace { +-struct mtp_acc_tracer { +- bool enabled = false; +- FILE * fp = nullptr; +- std::mutex mu; +- +- mtp_acc_tracer() { +- const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); +- if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { +- return; +- } +- fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); +- enabled = (fp != nullptr); +- } +- +- ~mtp_acc_tracer() { +- if (fp && fp != stderr) { +- std::fclose(fp); +- } +- } +- +- void writeln(const std::string & line) { +- if (!enabled) { +- return; +- } +- std::lock_guard lk(mu); +- std::fputs(line.c_str(), fp); +- std::fputc('\n', fp); +- std::fflush(fp); +- } +-}; +- +-mtp_acc_tracer & mtp_tracer() { +- static mtp_acc_tracer t; +- return t; +-} +-} // namespace +- +-struct common_speculative_impl_draft_assistant : public common_speculative_impl { +- common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) +- +- llama_context * ctx_tgt = nullptr; +- +- int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes +- int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row +- +- // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows +- // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds +- // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h +- // scheme so multi-seq (PolyKV) works without per-seq server plumbing. +- std::vector> pending_h; // [n_seq][n_embd_backbone] +- std::vector> verify_h; // [n_seq][n_rows * n_embd_backbone] +- std::vector verify_h_rows; +- std::vector i_batch_beg; +- std::vector i_batch_end; +- +- // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head +- // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain +- // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. +- int skip_streak_threshold = 0; +- std::vector zero_accept_streak; +- std::vector skip_last_draft; +- std::vector prev_n_acc_at_draft; +- +- int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change +- +- static float compute_h_l2(const float * h, int32_t n) { +- if (!h || n <= 0) { +- return 0.0f; +- } +- double s = 0.0; +- for (int32_t i = 0; i < n; ++i) { +- s += (double) h[i] * (double) h[i]; +- } +- return (float) std::sqrt(s); +- } +- +- common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) +- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) +- , params(params.draft) +- { +- ctx_tgt = this->params.ctx_tgt; +- GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); +- +- const llama_model * model_tgt = llama_get_model(ctx_tgt); +- n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); +- n_embd_out = llama_model_n_embd_out(model_tgt); +- +- LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); +- LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, +- this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); +- +- // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. +- llama_set_embeddings(ctx_tgt, true); +- +- if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { +- const int v = std::atoi(e); +- if (v >= 1 && v <= 32) { +- skip_streak_threshold = v; +- } +- } +- +- pending_h.assign(n_seq, std::vector((size_t) std::max(1, n_embd_backbone), 0.0f)); +- verify_h.assign(n_seq, {}); +- verify_h_rows.assign(n_seq, 0); +- i_batch_beg.assign(n_seq, -1); +- i_batch_end.assign(n_seq, -1); +- zero_accept_streak.assign(n_seq, 0); +- skip_last_draft.assign(n_seq, 0); +- prev_n_acc_at_draft.assign(n_seq, 0); +- } +- +- ~common_speculative_impl_draft_assistant() override = default; +- +- void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { +- llama_set_embeddings(ctx_tgt, true); +- if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { +- skip_last_draft[seq_id] = 0; +- zero_accept_streak[seq_id] = 0; +- } +- } +- +- // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() +- // can select the last-accepted token's hidden state. The spec framework calls process() after the +- // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which +- // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. +- // NOTE (bug-858): PRE-norm harvest was TESTED and REFUTED — it collapses accept to ~0.05 at ALL +- // positions (incl short ctx). This drafter expects the post-norm hidden; do NOT swap to pre_norm. +- bool process(const llama_batch & batch_in) override { +- if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { +- return true; +- } +- if (n_embd_backbone <= 0) { +- return true; +- } +- +- const int32_t n_tokens = batch_in.n_tokens; +- std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); +- std::fill(i_batch_end.begin(), i_batch_end.end(), -1); +- +- for (int k = 0; k < n_tokens; ++k) { +- const llama_seq_id s = batch_in.seq_id[k][0]; +- if (s < 0 || s >= (llama_seq_id) n_seq) { +- continue; +- } +- i_batch_end[s] = k; +- if (i_batch_beg[s] < 0) { +- i_batch_beg[s] = k; +- } +- } +- +- const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); +- const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); +- +- for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { +- if (i_batch_beg[s] < 0) { +- continue; +- } +- const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; +- verify_h_rows[s] = n_rows; +- verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); +- +- for (int32_t i = 0; i < n_rows; ++i) { +- const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); +- if (h) { +- std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, +- (size_t) n_copy * sizeof(float)); +- } +- } +- // default pending_h to the last row (correct when all drafts accepted / on prefill) +- std::memcpy(pending_h[s].data(), +- verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); +- } +- +- return true; +- } +- +- void draft(common_speculative_draft_params_vec & dparams) override { +- if (n_embd_backbone <= 0) { +- return; +- } +- +- // bug-858: draft depth follows --spec-draft-n-max (n_max) and the room left in context +- // (dp.n_max = n_draft_max), NOT draft_block_size. The MTP head re-runs autoregressively per +- // step (decode_mtp_sync/fused build a fresh single-step graph each step: argmax -> next +- // last_token, h_post -> next h_prev), so depth is bounded by n_max, not a fixed block. The +- // legacy draft_block_size-1 ceiling (default 3 -> 2) hard-capped ours at 2 drafts/round and +- // blocked n-max scaling vs upstream b9859 (which drafts to n_max autoregressively) -> REMOVED. +- +- for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { +- auto & dp = dparams[seq_id]; +- if (!dp.drafting) { +- continue; +- } +- +- // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, +- // the previous batch produced 0 accepted drafts. +- if (skip_last_draft[seq_id]) { +- skip_last_draft[seq_id] = 0; +- } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { +- zero_accept_streak[seq_id]++; +- } else { +- zero_accept_streak[seq_id] = 0; +- } +- if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { +- zero_accept_streak[seq_id] = 0; +- skip_last_draft[seq_id] = 1; +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- continue; // empty result -> server falls back to single-token verify +- } +- +- int32_t n_steps = params.n_max > 0 ? params.n_max : 1; +- if (dp.n_max > 0) { +- n_steps = std::min(n_steps, dp.n_max); +- } +- if (n_steps <= 0) { +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- continue; +- } +- +- float * h_prev = pending_h[seq_id].data(); +- +- llama_memory_t mem = llama_get_memory(ctx_tgt); +- llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; +- if (attn_pos < 0) { +- attn_pos = 0; +- } +- +- std::vector out((size_t) n_steps, 0); +- const int32_t rc = llama_decode_mtp( +- ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, +- /*out_drafts =*/ out.data(), +- /*out_logits =*/ nullptr, +- /*out_h_prev_last=*/ nullptr); +- +- prev_n_acc_at_draft[seq_id] = n_acc_drafts; +- +- if (rc != 0) { +- LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); +- continue; +- } +- +- auto & result = *dp.result; +- for (int32_t i = 0; i < n_steps; ++i) { +- result.push_back(out[i]); +- } +- +- if (mtp_tracer().enabled) { +- std::ostringstream oss; +- oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter +- << ",\"seq_id\":" << (int) seq_id +- << ",\"id_last\":" << (int) dp.id_last +- << ",\"attn_pos\":" << (int) attn_pos +- << ",\"n_steps\":" << n_steps +- << ",\"h_l2\":" << std::fixed << std::setprecision(4) +- << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; +- for (int32_t i = 0; i < n_steps; ++i) { +- if (i) oss << ','; +- oss << (int) out[i]; +- } +- oss << "]}"; +- mtp_tracer().writeln(oss.str()); +- } +- } +- } +- +- void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { +- if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { +- return; +- } +- const int32_t n_rows = verify_h_rows[seq_id]; +- if (n_rows <= 0 || n_embd_backbone <= 0) { +- return; +- } +- // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the +- // last-accepted token's hidden state (clamped to the harvested rows). +- const int32_t i_h = std::min(n_accepted, n_rows - 1); +- std::memcpy(pending_h[seq_id].data(), +- verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, +- (size_t) n_embd_backbone * sizeof(float)); +- +- if (mtp_tracer().enabled) { +- std::ostringstream oss; +- oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter +- << ",\"seq_id\":" << (int) seq_id +- << ",\"n_accepted\":" << (int) n_accepted << "}"; +- mtp_tracer().writeln(oss.str()); +- ++trace_iter; +- } +- } +- +- bool need_embd() const override { +- return true; // MTP reads POST-norm output embeddings (h_prev) +- } +-}; + + struct common_speculative { + common_speculative_draft_params_vec dparams; +@@ -1934,15 +1625,15 @@ common_speculative * common_speculative_init(common_params_speculative & params, + break; + } + case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter +- // opencoti bug-858 dual-context MTP: when the server created a real ctx_dft FROM the +- // assistant model (ctx_other=ctx_tgt; --spec-type draft-assistant + OPENCOTI_MTP_DUAL_CTX), +- // run the shared-KV draft_mtp driver (upstream b9859 is_mem_shared). Otherwise fall +- // back to the proven single-context in-target assistant (ctx_dft == nullptr). +- if (config.params.draft.ctx_dft != nullptr) { +- impls.push_back(std::make_unique(config.params, n_seq)); +- } else { +- impls.push_back(std::make_unique(config.params, n_seq)); ++ // opencoti bug-858 dual-context MTP (#611): the assistant ALWAYS runs as a real ++ // ctx_dft created FROM the assistant model (ctx_other=ctx_tgt) through the shared-KV ++ // draft_mtp driver (upstream b9859 is_mem_shared). The single-context in-target ++ // facade was removed in #611 after the dual-context port proved accept parity. ++ if (config.params.draft.ctx_dft == nullptr) { ++ LOG_ERR("%s: draft-assistant requires the dual-context draft (server creates ctx_dft with ctx_other=target)\n", __func__); ++ return nullptr; + } ++ impls.push_back(std::make_unique(config.params, n_seq)); + break; + } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { +@@ -2010,8 +1701,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, + // opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / + // set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative + // framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ +-// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on +-// common_speculative_impl_draft_assistant above). ++// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete. + + void common_speculative_free(common_speculative * spec) { + if (spec == nullptr) { +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +index e6517be..2f4da86 100644 +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -1041,22 +1041,6 @@ extern "C" { + struct llama_context * ctx, + struct llama_batch batch); + +- // opencoti F5 M6-S4 mtp +- // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, +- // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). +- // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) +- // receive per-step logits and the final backbone hidden. Returns 0 on success. +- LLAMA_API int32_t llama_decode_mtp( +- struct llama_context * ctx, +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- + // opencoti fused-NextN (#590): fuse N per-step NextN draft decodes into one graph on a + // DECODER_MTP (Qwen NextN) draft context. GREEDY-ONLY (no p_min early-stop). Returns 0 on + // success. last_token must already be resident in ctx's cache at attn_pos (seed decode first). +diff --git a/llama.cpp/src/llama-arch.h b/llama.cpp/src/llama-arch.h +index 9aef864..85b1e98 100644 +--- a/llama.cpp/src/llama-arch.h ++++ b/llama.cpp/src/llama-arch.h +@@ -570,8 +570,8 @@ enum llm_tensor { + + // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 + // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile +- // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our +- // single-context build_attn_mtp cross-read — NOT upstream's ctx_other. See plan. ++ // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine is the ++ // bug-858 dual-context port (cparams.ctx_other, upstream-shaped). See plan. + LLM_TENSOR_NEXTN_PROJ_PRE, + LLM_TENSOR_NEXTN_PROJ_POST, + LLM_TENSOR_MASKED_EMBD_CENTROIDS, +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +index 7d27212..1579acc 100644 +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -9,7 +9,7 @@ + #include "llama-memory.h" + #include "llama-mmap.h" + #include "llama-model.h" +-#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp ++#include "llama-kv-cache-iswa.h" + #include "llama-ext.h" + #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" +@@ -527,17 +527,6 @@ llama_context::llama_context( + } + + llama_context::~llama_context() { +- // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down +- // sched_mtp and the backends it touches. +- if (mtp_worker.joinable()) { +- { +- std::lock_guard lk(mtp_mu); +- mtp_worker_stop.store(true, std::memory_order_release); +- } +- mtp_cv_request.notify_all(); +- mtp_worker.join(); +- } +- + if (!model.hparams.no_alloc) { + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; +@@ -1421,208 +1410,6 @@ bool llama_context::set_adapter_cvec( + return res; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-bool llama_context::ensure_sched_mtp() { +- // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count +- // (mtp_fused_steps). A fused N-step graph has ~N× the nodes of the single-step graph and a +- // distinct topology, so when a larger block is requested we drop and rebuild the reserve. +- const int32_t need = std::max(mtp_fused_steps, 1); +- if (sched_mtp && mtp_reserved_steps >= need) { +- return true; +- } +- if (!model.mtp_assistant) { +- return false; +- } +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- +- const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); +- const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; +- +- gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); +- sched_mtp.reset(ggml_backend_sched_new( +- backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), +- max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); +- if (!sched_mtp) { +- LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. +- // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), +- // so a single reserve covers all subsequent decode_mtp calls. +- { +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- llama_memory_context_ptr mctx = memory->init_full(); +- if (!mctx) { +- LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(0); +- data->seq_idx[0] = 0; +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = 0; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- const uint32_t save_n_outputs = n_outputs; +- n_outputs = 1; +- +- auto * res = gf_res_prev_mtp.get(); +- const auto gparams = graph_params_mtp(res, ub, mctx.get()); +- res->reset(); +- ggml_backend_sched_reset(sched_mtp.get()); +- +- auto * gf = model.build_graph(gparams); +- n_outputs = save_n_outputs; +- +- if (!gf) { +- LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { +- LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); +- sched_mtp.reset(); +- gf_res_prev_mtp.reset(); +- return false; +- } +- // Discard the reserve graph cache so the first real call rebuilds with proper inputs. +- res->reset(); +- ggml_backend_sched_reset(sched_mtp.get()); +- } +- +- // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in +- // decode_mtp_async (only when the opt-in async path is actually taken), NOT here — so the +- // default in-thread sync path creates no extra thread and is runtime-identical to P3. +- +- mtp_reserved_steps = need; // reserve now covers `need` fused steps +- return true; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-llm_graph_result * llama_context::process_ubatch_mtp( +- const llama_ubatch & ubatch, +- llama_memory_context_i * mctx, +- ggml_status & ret) { +- GGML_ASSERT(sched_mtp && gf_res_prev_mtp); +- +- auto * res = gf_res_prev_mtp.get(); +- auto * gf = res->get_gf(); +- +- // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it +- // explicitly here in case a concurrent target decode mutates it. The MTP graph +- // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. +- llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); +- gparams.n_outputs = 1; +- gparams.sched = sched_mtp.get(); +- +- if (!graph_reuse_disable && res->can_reuse(gparams)) { +- // sched_mtp does not use pipeline parallelism (created with pipeline=false), +- // so no synchronize is needed before set_inputs. +- } else { +- res->reset(); +- +- ggml_backend_sched_reset(sched_mtp.get()); +- ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); +- +- gf = model.build_graph(gparams); +- if (!gf) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); +- ret = GGML_STATUS_FAILED; +- return nullptr; +- } +- if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { +- LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); +- ret = GGML_STATUS_ALLOC_FAILED; +- return nullptr; +- } +- } +- +- res->set_inputs(&ubatch); +- +- const auto status = graph_compute_mtp(res->get_gf()); +- +- if (status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); +- ret = status; +- return nullptr; +- } +- +- ret = GGML_STATUS_SUCCESS; +- return res; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { +- // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s +- // threadpool/n_threads dance only for the CPU backend; the MTP head is small and +- // thread saturation matters less, but consistency with graph_compute() avoids +- // surprising backend reconfigurations between target and MTP runs. +- // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker +- // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. +- const int n_threads = cparams.n_threads; +- ggml_threadpool_t tp = threadpool; +- +- if (backend_cpu != nullptr) { +- auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); +- auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) +- ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); +- if (set_threadpool_fn) { +- set_threadpool_fn(backend_cpu, tp); +- } +- } +- for (const auto & set_n_threads_fn : set_n_threads_fns) { +- set_n_threads_fn.second(set_n_threads_fn.first, n_threads); +- } +- +- auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); +- if (status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", +- __func__, status); +- } +- return status; +-} +- + llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { + if (mctx && !mctx->apply()) { + LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); +@@ -2093,14 +1880,6 @@ int llama_context::decode(const llama_batch & batch_inp) { + + bool did_optimize = false; + +- // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its +- // KV below (memory_update's pending shifts/copies, then init_batch's slot apply — the S1 +- // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this +- // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in +- // the current sync-facade integration (nothing is in flight here) and a single pointer +- // check when no MTP assistant is loaded. +- mtp_drain_before_mutate(); +- + // handle any pending shifts/copies + memory_update(false); + +@@ -2684,368 +2463,11 @@ ggml_cgraph * llama_context::graph_reserve( + return gf; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-llm_graph_params llama_context::graph_params_mtp( +- llm_graph_result * res, +- const llama_ubatch & ubatch, +- const llama_memory_context_i * mctx) const { +- const llama_model * mtp = model.mtp_assistant.get(); +- GGML_ASSERT(mtp); +- +- return { +- /*.arch =*/ mtp->arch, +- /*.hparams =*/ mtp->hparams, +- /*.cparams =*/ cparams, +- /*.ubatch =*/ ubatch, +- /*.gtype =*/ LLM_GRAPH_TYPE_MTP, +- /*.sched =*/ sched.get(), +- /*.backend_cpu =*/ backend_cpu, +- /*.cvec =*/ cvec.get(), +- /*.loras =*/ loras.get(), +- /*.mctx =*/ mctx, +- /*.cross =*/ &cross, +- /*.samplers =*/ sampling.samplers, +- /*.n_outputs =*/ n_outputs, +- /*.cb =*/ graph_get_cb(), +- /*.res =*/ res, +- /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) +- }; +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. +-int32_t llama_context::decode_mtp( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams +- // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the +- // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes async→wait. +- // With no target/draft overlap this is byte-identical to the P3 sync path — the worker runs +- // the same N-step loop off-thread while we block in decode_mtp_wait. +- if (out_logits) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_logits, out_h_prev_last); +- } +- // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the +- // fused single-graph draft (decode_mtp_fused) — ONE launch+sync+D2H for the whole block, the +- // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused +- // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse → sequential. +- static const bool fused_enabled = [] { +- const char * e = getenv("OPENCOTI_MTP_FUSED"); +- return !(e && e[0] == '0'); +- }(); +- if (fused_enabled && n_steps > 1) { +- return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_h_prev_last); +- } +- // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). +- // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker +- // gives NO throughput benefit — real depth-2 overlap is deferred (limited by the h-prev +- // dependency; even the AtomicBot fork ships depth-1) — AND the worker-thread CUDA execution +- // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off +- // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. +- static const bool async_enabled = [] { +- const char * e = getenv("OPENCOTI_MTP_ASYNC"); +- return e && e[0] == '1'; +- }(); +- if (!async_enabled) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, out_logits, out_h_prev_last); +- } +- const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); +- if (rc != 0) { +- return rc; +- } +- return decode_mtp_wait(out_drafts, out_h_prev_last); +-} +- +-// opencoti F5 M6-S4 mtp (P2 inert) +-int32_t llama_context::decode_mtp_sync( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- +- // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, +- // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) +- mtp_fused_steps = 1; +- +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- +- const int32_t n_vocab = model.vocab.n_tokens(); +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0) { +- LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); +- return -4; +- } +- +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(seq_id); +- data->seq_idx[(size_t) seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post +- // back). It MUST be flagged output=1 so build_inp_out_ids selects its row — and so the live +- // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here +- // produced a divergent graph whose output-selection tensor was left with a null backend +- // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. +- data->output[0] = 1; +- +- if (n_steps <= 0) { +- return 0; +- } +- +- // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token +- // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. +- for (int32_t k = 0; k < n_steps; ++k) { +- data->token[0] = last_token; +- // bug-858: the gemma4-assistant (is_mem_shared) drafts ALL steps from the SAME query +- // position — the head is trained to predict multiple future tokens from one position, with +- // the recurrence carried by the h_prev hidden chain (NOT the RoPE angle). Incrementing the +- // position per step (attn_pos+1+k) hands the head a mistrained RoPE angle from step 1 on, +- // degrading step-1+ drafts (aggregate accept ~0.69 vs upstream 0.79). Match b9859 draft_mtp +- // is_mem_shared: common_batch_add(batch, id, dp.n_past, ...) — constant dp.n_past = attn_pos+1. +- data->pos[0] = attn_pos + 1; +- std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); +- +- int32_t best_i32 = 0; +- ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); +- out_drafts[k] = (llama_token) best_i32; +- +- if (out_logits) { +- ggml_tensor * t_logits = res->get_logits(); +- GGML_ASSERT(t_logits); +- ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, +- 0, (size_t) n_vocab * sizeof(float)); +- } +- +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); +- +- last_token = (llama_token) best_i32; +- } +- +- if (out_h_prev_last) { +- std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); +- } +- +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head +-// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls +-// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index — already +-// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. +-// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, +-// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). +-// +-// One mask shared across steps (built from step 0's position). For FULL-attn target layers and +-// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa — the short-context / +-// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are +-// byte-identical to the sequential path. For SWA layers at long context the true per-step window +-// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is +-// output-safe — the target verify pass rejects any mismatched draft, so the emitted stream is +-// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix +-// is a per-step mask slice — deferred until measured. See buglog bug-420.) +-int32_t llama_context::decode_mtp_fused( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_h_prev_last) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- if (n_steps <= 0) { +- return 0; +- } +- // A single step has nothing to fuse — defer to the proven sequential path. +- if (n_steps == 1) { +- return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, +- out_drafts, /*out_logits=*/nullptr, out_h_prev_last); +- } +- +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0) { +- LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); +- return -4; +- } +- +- // Build/reserve a fused n_steps graph. +- mtp_fused_steps = n_steps; +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- +- // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step +- // query positions consumed by the wrapper's inp_pos_steps (set_input). +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(n_steps); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(seq_id); +- data->seq_idx[(size_t) seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = (uint32_t) n_steps; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; +- +- data->token[0] = last_token; +- std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); +- // bug-858: constant query position for ALL draft steps (see decode_mtp_sync note). The +- // gemma4-assistant (is_mem_shared) is trained to predict every future draft token from the +- // SAME position (dp.n_past = attn_pos+1); recurrence lives in the h_prev chain, not the RoPE +- // angle. The old attn_pos+1+k mistrained step-1+ RoPE -> accept 0.69 vs upstream 0.79. +- for (int32_t k = 0; k < n_steps; ++k) { +- data->pos[k] = attn_pos + 1; +- } +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); +- GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); +- +- std::vector drafts((size_t) n_steps); +- ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); +- for (int32_t k = 0; k < n_steps; ++k) { +- out_drafts[k] = (llama_token) drafts[(size_t) k]; +- } +- +- if (out_h_prev_last) { +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); +- } +- +- return 0; +-} +- + // opencoti fused-NextN (#590 / bug-858): fuse the N per-step NextN draft decodes into ONE graph on + // the draft context (ctx_dft, type LLAMA_CONTEXT_TYPE_MTP → LLM_GRAPH_TYPE_DECODER_MTP). This is the + // throughput fix: the un-fused path pays a host sync per draft step; here one graph emits N greedy +-// drafts with a single sync. Twin of decode_mtp_fused (Gemma assistant) but for the UNIFIED cache — +-// so no mtp_assistant / sched_mtp / kv_iswa. The N fused steps read-only cross-attend the frozen ++// drafts with a single sync, for the UNIFIED cache — ++// so no mtp_assistant / kv_iswa involvement. The N fused steps read-only cross-attend the frozen + // prefix (build_attn_readonly_nextn, Option A); recurrence is carried by the on-device hidden chain. + // GREEDY-ONLY: no p_min early-stop (the caller reconciles draft length). See + // docs/features/fused_nextn_mtp.md. Returns 0 on success, negative on error. +@@ -3062,7 +2484,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + return -2; + } + // NextN self-spec targets are full-attention → unified cache. (iSWA NextN is not a thing; +- // the Gemma assistant path is decode_mtp_fused.) ++ // the Gemma assistant runs as a dual context via draft_mtp, not through this driver.) + auto * kv = dynamic_cast(memory.get()); + if (!kv) { + LLAMA_LOG_ERROR("%s: fused NextN requires a unified llama_kv_cache\n", __func__); +@@ -3133,7 +2555,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + } + + // Build/compute the fused N-step graph on the NORMAL sched (graph_params sets n_mtp_steps=N for +- // DECODER_MTP). Inlined like process_ubatch_mtp — deliberately NOT process_ubatch, whose apply() ++ // DECODER_MTP). Inlined graph processing — deliberately NOT process_ubatch, whose apply() + // would overwrite the pmax prefix cell. + mtp_fused_steps = n_steps; + +@@ -3187,7 +2609,7 @@ int32_t llama_context::decode_mtp_fused_nextn( + } + res->set_inputs(&ub); + +- // Compute on the dedicated sched (mirror graph_compute_mtp's CPU-threadpool dance). ++ // Compute on the dedicated sched (with the CPU-threadpool attach/set dance). + if (backend_cpu != nullptr) { + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + auto * set_tp_fn = (decltype(ggml_backend_cpu_set_threadpool) *) +@@ -3227,224 +2649,6 @@ int32_t llama_context::decode_mtp_fused_nextn( + return 0; + } + +-// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one +-// in-flight request per context (returns -7 if a prior request was not yet waited). +-int32_t llama_context::decode_mtp_async( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- const float * h_prev, +- int32_t n_steps) { +- if (!model.mtp_assistant) { +- LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); +- return -1; +- } +- if (!memory) { +- LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); +- return -2; +- } +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- if (n_bb == 0 || !h_prev || n_steps <= 0) { +- LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", +- __func__, n_bb, (const void *) h_prev, n_steps); +- return -4; +- } +- if (!ensure_sched_mtp()) { +- LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); +- return -8; +- } +- if (!mtp_worker.joinable()) { +- mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); +- } +- +- { +- std::unique_lock lk(mtp_mu); +- if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { +- LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", +- __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, +- (int) mtp_completed.has_value()); +- return -7; +- } +- mtp_request req; +- req.seq_id = seq_id; +- req.attn_pos = attn_pos; +- req.last_token = last_token; +- req.n_steps = n_steps; +- req.h_prev.assign(h_prev, h_prev + n_bb); +- mtp_pending = std::move(req); +- } +- mtp_cv_request.notify_one(); +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last +-// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). +-int32_t llama_context::decode_mtp_wait( +- llama_token * out_drafts, +- float * out_h_prev_last) { +- std::unique_lock lk(mtp_mu); +- mtp_cv_response.wait(lk, [this] { +- return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); +- }); +- if (!mtp_completed.has_value()) { +- LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); +- return -7; +- } +- mtp_response resp = std::move(*mtp_completed); +- mtp_completed.reset(); +- lk.unlock(); +- +- if (resp.status != 0) { +- return resp.status; +- } +- if (out_drafts && !resp.drafts.empty()) { +- std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); +- } +- if (out_h_prev_last && !resp.h_prev_last.empty()) { +- std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); +- } +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's +-// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the +-// single ubatch row MUST be flagged output — a 0 here reintroduces the null-out-ids abort +-// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. +-int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { +- auto * kv_iswa = dynamic_cast(memory.get()); +- if (!kv_iswa) { +- LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); +- return -3; +- } +- const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; +- +- auto data = std::make_shared(); +- data->token.resize(1); +- data->embd.resize(n_bb); +- data->pos.resize(1); +- data->n_seq_id.resize(1); +- data->seq_id.resize(1); +- data->seq_id_data.resize(1); +- data->output.resize(1); +- data->seq_idx.resize(LLAMA_MAX_SEQ, -1); +- data->seq_id_unq.push_back(req.seq_id); +- data->seq_idx[(size_t) req.seq_id] = 0; +- +- llama_ubatch ub{}; +- ub.b_equal_seqs = 1; +- ub.n_tokens = 1; +- ub.n_seq_tokens = 1; +- ub.n_seqs = 1; +- ub.n_seqs_unq = 1; +- ub.n_pos = 1; +- ub.token = data->token.data(); +- ub.embd = data->embd.data(); +- ub.pos = data->pos.data(); +- ub.n_seq_id = data->n_seq_id.data(); +- ub.seq_id = data->seq_id.data(); +- ub.seq_id_unq = data->seq_id_unq.data(); +- ub.seq_idx = data->seq_idx.data(); +- ub.output = data->output.data(); +- ub.data = data; +- +- data->n_seq_id[0] = 1; +- data->seq_id_data[0] = req.seq_id; +- data->seq_id[0] = &data->seq_id_data[0]; +- data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 +- +- std::vector h(req.h_prev); +- llama_token last_token = req.last_token; +- resp.drafts.assign((size_t) req.n_steps, 0); +- +- for (int32_t k = 0; k < req.n_steps; ++k) { +- data->token[0] = last_token; +- data->pos[0] = req.attn_pos + 1 + (llama_pos) k; +- std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); +- +- llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); +- if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); +- return -5; +- } +- +- ggml_status status = GGML_STATUS_SUCCESS; +- llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); +- if (!res || status != GGML_STATUS_SUCCESS) { +- LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); +- return -6; +- } +- +- ggml_backend_sched_synchronize(sched_mtp.get()); +- +- ggml_tensor * t_arg = res->get_argmax(); +- GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); +- int32_t best_i32 = 0; +- ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); +- last_token = (llama_token) best_i32; +- resp.drafts[(size_t) k] = last_token; +- +- ggml_tensor * t_post = res->get_embd(); +- GGML_ASSERT(t_post); +- ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); +- } +- +- resp.h_prev_last = std::move(h); +- return 0; +-} +- +-// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on +-// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. +-void llama_context::mtp_worker_loop() { +- for (;;) { +- mtp_request req; +- { +- std::unique_lock lk(mtp_mu); +- mtp_cv_request.wait(lk, [this] { +- return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); +- }); +- if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { +- return; +- } +- req = std::move(*mtp_pending); +- mtp_pending.reset(); +- mtp_in_flight = true; +- } +- +- mtp_response resp; +- resp.status = decode_mtp_run(req, resp); +- +- { +- std::lock_guard lk(mtp_mu); +- mtp_in_flight = false; +- mtp_completed = std::move(resp); +- } +- mtp_cv_response.notify_one(); +- } +-} +- +-// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV +-// read retired), then sync sched_mtp — WITHOUT consuming mtp_completed (the spec loop's _wait +-// still needs the drafts). Called at decode() top before any KV mutation. Inert when no +-// assistant is loaded or nothing is in flight (drained==false ⇒ no extra sched sync). +-void llama_context::mtp_drain_before_mutate() { +- if (!model.mtp_assistant) { +- return; +- } +- bool drained = false; +- { +- std::unique_lock lk(mtp_mu); +- if (mtp_pending.has_value() || mtp_in_flight) { +- mtp_cv_response.wait(lk, [this] { +- return !mtp_in_flight && !mtp_pending.has_value(); +- }); +- drained = true; +- } +- } +- if (drained && sched_mtp) { +- ggml_backend_sched_synchronize(sched_mtp.get()); +- } +-} +- + llm_graph_params llama_context::graph_params( + llm_graph_result * res, + const llama_ubatch & ubatch, +@@ -5177,24 +4381,6 @@ int32_t llama_decode( + return ret; + } + +-// opencoti F5 M6-S4 mtp (P2 inert) +-int32_t llama_decode_mtp( +- llama_context * ctx, +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last) { +- if (!ctx) { +- LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); +- return -1; +- } +- return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); +-} +- + int32_t llama_decode_mtp_fused_nextn( + llama_context * ctx, + llama_seq_id seq_id, +diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h +index b6b0efb..32034fd 100644 +--- a/llama.cpp/src/llama-context.h ++++ b/llama.cpp/src/llama-context.h +@@ -147,48 +147,8 @@ struct llama_context { + llama_memory_context_i * mctx, + ggml_status & ret); + +- // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. +- llm_graph_params graph_params_mtp( +- llm_graph_result * res, +- const llama_ubatch & ubatch, +- const llama_memory_context_i * mctx) const; +- +- int32_t decode_mtp( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- +- int32_t decode_mtp_sync( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_logits, +- float * out_h_prev_last); +- +- // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps +- // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft +- // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced +- // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits +- // verification path stays on decode_mtp_sync. +- int32_t decode_mtp_fused( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- float * h_prev, +- int32_t n_steps, +- llama_token * out_drafts, +- float * out_h_prev_last); +- + // opencoti fused-NextN (#590): fused N-step greedy draft on a DECODER_MTP (Qwen NextN) draft +- // context using the unified cache. Twin of decode_mtp_fused minus mtp_assistant/sched_mtp/iSWA. ++ // context using the unified cache (the only in-context MTP draft driver). + int32_t decode_mtp_fused_nextn( + llama_seq_id seq_id, + llama_pos attn_pos, +@@ -198,25 +158,6 @@ struct llama_context { + llama_token * out_drafts, + float * out_h_prev_last); + +- // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync +- // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams +- // no per-step logits); otherwise it submits via decode_mtp_async and blocks in +- // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the +- // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). +- int32_t decode_mtp_async( +- llama_seq_id seq_id, +- llama_pos attn_pos, +- llama_token last_token, +- const float * h_prev, +- int32_t n_steps); +- int32_t decode_mtp_wait( +- llama_token * out_drafts, +- float * out_h_prev_last); +- // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before +- // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP +- // assistant is loaded or nothing is in flight (the common case). +- void mtp_drain_before_mutate(); +- + int encode(const llama_batch & batch_inp); + int decode(const llama_batch & batch_inp); + +@@ -333,14 +274,6 @@ private: + const llama_memory_context_i * mctx, + llm_graph_type gtype) const; + +- // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. +- bool ensure_sched_mtp(); +- llm_graph_result * process_ubatch_mtp( +- const llama_ubatch & ubatch, +- llama_memory_context_i * mctx, +- ggml_status & ret); +- ggml_status graph_compute_mtp(ggml_cgraph * gf); +- + llm_graph_cb graph_get_cb() const; + + // TODO: read/write lora adapters and cvec +@@ -440,58 +373,22 @@ private: + llm_graph_result_ptr gf_res_prev; + llm_graph_result_ptr gf_res_reserve; + +- // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be +- // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable +- // single-token MTP graph result across steps of one decode_mtp call. +- ggml_backend_sched_ptr sched_mtp; +- llm_graph_result_ptr gf_res_prev_mtp; +- + // opencoti #590/bug-858: DEDICATED scheduler + result for the fused NextN self-spec draft + // (decode_mtp_fused_nextn). It must NOT share the main sched/gf_res_prev with target decode: + // every interleaved target decode RESETS the main sched (reallocating compute buffers) and + // gf_res_prev, so a fused graph parked there can never satisfy can_reuse AND its tensor memory is + // clobbered → 100% rebuild + CUDA-recapture per draft round (measured: build 0.5ms + comp-launch +- // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched (like the Gemma assistant's +- // sched_mtp, but ensure_sched_mtp is assistant-only: needs mtp_assistant + iswa cache) isolates the ++ // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched isolates the + // fused graph so can_reuse compares fused-vs-fused (stable within a 256-cell n_kv bucket) and the + // ggml graph + CUDA capture persist across rounds. NextN and Gemma-assistant are mutually exclusive. + ggml_backend_sched_ptr sched_nextn; + llm_graph_result_ptr gf_res_prev_nextn; + int32_t nextn_reserved_steps = 0; + +- // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for +- // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it +- // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so +- // ensure_sched_mtp only re-reserves when a larger fused block is requested. ++ // opencoti #590 fused NextN: number of draft steps to unroll into the MTP graph for the ++ // NEXT build (decode_mtp_fused_nextn sets it to n_steps around its build; the normal ++ // decode/reserve paths leave it at 1 so graph_params stays single-step). + int32_t mtp_fused_steps = 1; +- int32_t mtp_reserved_steps = 0; +- +- // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. +- // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp +- // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync +- // facade), so the worker never races the target's sched. The destructor joins the worker. +- struct mtp_request { +- llama_seq_id seq_id = 0; +- llama_pos attn_pos = 0; +- llama_token last_token = 0; +- std::vector h_prev; +- int32_t n_steps = 0; +- }; +- struct mtp_response { +- int32_t status = 0; +- std::vector drafts; +- std::vector h_prev_last; +- }; +- std::thread mtp_worker; +- std::atomic mtp_worker_stop{false}; +- std::mutex mtp_mu; +- std::condition_variable mtp_cv_request; +- std::condition_variable mtp_cv_response; +- std::optional mtp_pending; // submitted, not yet picked up +- bool mtp_in_flight = false; // worker is processing +- std::optional mtp_completed; // worker finished, awaiting _wait +- int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop +- void mtp_worker_loop(); + + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_ptr buf_output; +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +index e0d712e..6fc9679 100644 +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -124,7 +124,7 @@ void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { + ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); + + // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] +- // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. ++ // (= attn_pos + 1 + k, filled by decode_mtp_fused_nextn). Single-step path leaves this empty. + // bug-870: mrope archs (qwen35) size each step's pos tensor I32[4] — fill the M-RoPE + // text-token layout [p,p,p,0] (mirrors llm_graph_input_pos::set_input); standard rope = [p]. + for (size_t k = 0; k < inp_pos_steps.size(); ++k) { +@@ -570,7 +570,7 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { + void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { + // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently — + // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP +- // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but ++ // dual-ctx assistant draft graph shares the target KV through mem_other, but + // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) + // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the + // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP +@@ -2999,95 +2999,9 @@ ggml_tensor * llm_graph_context::build_attn( + return cur; + } + +-// opencoti F5 M6-S4 mtp +-ggml_tensor * llm_graph_context::build_attn_mtp( +- llm_graph_input_attn_kv_iswa * inp, +- ggml_tensor * wo, +- ggml_tensor * wo_b, +- ggml_tensor * q_cur, +- ggml_tensor * kq_b, +- ggml_tensor * sinks, +- ggml_tensor * v_mla, +- float kq_scale, +- int il_mtp, +- int32_t il_kv_tgt, +- bool read_from_swa_kv, +- int64_t kv_embd_head_v, +- int64_t kv_n_head_v, +- bool use_k_as_v) const { +- auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; +- auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; +- +- if (k_rot) { +- q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); +- } +- if (v_rot) { +- // V-only rotation is applied after MHA in the standard path; no V write here. +- } +- +- ggml_build_forward_expand(gf, q_cur); +- +- const auto * mctx_iswa = inp->mctx; +- const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); +- +- const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); +- +- ggml_tensor * q = q_cur; +- ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); +- ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); +- +- if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0 +- || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) +- // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the +- // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp +- // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at +- // head_dim∈{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 +- // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired +- // inverse on the output → wrong logits (graph still *built*, which is why P2's inert build +- // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity +- // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. +- if (q->ne[0] % 128 != 0) { +- const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; +- q = ggml_pad(ctx0, q, pad, 0, 0, 0); +- } +- if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } +- } +- +- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); +- cb(cur, "kqv_out_mtp", il_mtp); +- +- if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0 +- || v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) +- const int64_t orig_v_head = kv_embd_head_v; +- const int64_t padded_v_head = v->ne[0]; +- if (padded_v_head != orig_v_head) { +- const int64_t n_head_v = kv_n_head_v; +- const int64_t n_tokens_cur = cur->ne[1]; +- cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); +- cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, +- cur->nb[1], cur->nb[2], 0); +- cur = ggml_cont(ctx0, cur); +- cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); +- } +- } +- +- if (v_rot) { +- cur = ggml_mul_mat_aux(ctx0, cur, v_rot); +- } +- +- if (wo) { +- cur = build_lora_mm(wo, cur); +- cb(cur, "mtp_wo_out", il_mtp); +- } +- if (wo_b) { +- cur = ggml_add(ctx0, cur, wo_b); +- } +- +- return cur; +-} + + // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the frozen prefix +-// KV on the UNIFIED cache. Twin of build_attn_mtp for the non-iSWA llm_graph_input_attn_kv used by ++// KV on the UNIFIED cache via the non-iSWA llm_graph_input_attn_kv used by + // Qwen NextN drafters. Reads mctx->get_k/get_v(il) with the input's kq_mask — which exposes the + // full prefix 0..pmax because get_n_kv reports the cache's used extent, not the mtp_slot_info cell + // count. Writes NO draft KV; recurrence is carried by the hidden-state chain. wo is applied by the +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +index d3a3499..daa61b0 100644 +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -595,7 +595,7 @@ struct llm_graph_params { + // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. + // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone + // into the next step with no host round-trip / per-step GPU sync (the throughput fix). +- // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. ++ // Read by the fused NextN builders; default 1 keeps every non-MTP construction unchanged. + int32_t n_mtp_steps = 1; + + // return true if the "other" params would result in a graph with the same topology as with the current params +@@ -1021,28 +1021,8 @@ struct llm_graph_context { + int il, + bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) + +- // opencoti F5 M6-S4 mtp +- // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). +- // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). +- // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). +- ggml_tensor * build_attn_mtp( +- llm_graph_input_attn_kv_iswa * inp, +- ggml_tensor * wo, +- ggml_tensor * wo_b, +- ggml_tensor * q_cur, +- ggml_tensor * kq_b, +- ggml_tensor * sinks, +- ggml_tensor * v_mla, +- float kq_scale, +- int il_mtp, +- int32_t il_kv_tgt, +- bool read_from_swa_kv, +- int64_t kv_embd_head_v, +- int64_t kv_n_head_v, +- bool use_k_as_v) const; +- + // opencoti fused-NextN (Option A): read-only cross-attention into the frozen prefix KV on the +- // UNIFIED cache. Twin of build_attn_mtp for llm_graph_input_attn_kv (non-iSWA). Reads ++ // UNIFIED cache (llm_graph_input_attn_kv). Reads + // mctx->get_k/get_v(il) with the input's kq_mask (which exposes 0..pmax via get_n_kv's used + // extent), runs build_attn_mha, writes NO draft KV. wo is applied by the caller. + // See docs/features/fused_nextn_mtp.md. +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +index 9628df1..f2e95a0 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -247,21 +247,6 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_full() { + return std::make_unique(this); + } + +-// opencoti F5 M6-S4 mtp +-llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { +- llama_kv_cache::slot_info_vec_t sinfos_base; +- llama_kv_cache::slot_info_vec_t sinfos_swa; +- +- sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); +- sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); +- +- std::vector ubatches; +- ubatches.push_back(std::move(ubatch)); +- +- return std::make_unique( +- this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); +-} +- + llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); + } +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +index 06fd1ae..a7eec3d 100644 +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -77,8 +77,6 @@ public: + + llama_memory_context_ptr init_full() override; + +- // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). +- llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +index 4416d38..71bf0d5 100644 +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -5439,7 +5439,7 @@ llama_kv_cache_context::llama_kv_cache_context( + llama_kv_cache::slot_info_vec_t sinfos, + std::vector ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { + // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, +- // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from ++ // which builds its graph inline and never calls apply()) get a valid mask/K/V shape from + // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a + // view with a garbage dim-2 (observed ne[2]=2956782432 → ggml.c view-bounds assert). For + // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. +diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp +index cab4fa8..64ea519 100644 +--- a/llama.cpp/src/models/gemma4-assistant.cpp ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -4,380 +4,10 @@ + #include + #include + +-static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { +- p.arch = mtp_model.arch; +- p.hparams = mtp_model.hparams; +- p.gtype = LLM_GRAPH_TYPE_MTP; +- return p; +-} +- +-// Last layer in [range_start, range_end) whose attention type matches want_swa. +-static int32_t gemma4_mtp_kv_layer_last_in_range( +- const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { +- int32_t best = -1; +- if (range_start < 0) { +- range_start = 0; +- } +- if (range_end > (int32_t) tgt.n_layer) { +- range_end = (int32_t) tgt.n_layer; +- } +- for (int32_t il = range_start; il < range_end; ++il) { +- if (tgt.is_swa((uint32_t) il) == want_swa) { +- best = il; +- } +- } +- return best; +-} +- +-// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> +-// (post-projected backbone hidden, vocabulary logits, argmax token id). +-// +-// Used by both the single-step path (n_mtp_steps==1) and the chained path +-// (n_mtp_steps>1, called n_mtp_steps times within the same graph). +-// +-// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position +-// for this step's RoPE (the cross-attn mask is shared across steps because the +-// target's KV cache only contains positions ≤ attn_pos and all step positions +-// are > attn_pos, so causal/SWA admit all target cells uniformly). +-// +-// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too — required for greedy match with verify). +-// `out_argmax` (I32 [1]): greedy token id on-device. +-static void gemma4_mtp_build_one_step( +- const llm_graph_context & gctx, +- const llama_model & target, +- const llama_model & mtp, +- llm_graph_input_attn_kv_iswa * inp_attn, +- ggml_tensor * tok_step, // I32 [1] +- ggml_tensor * h_step, // F32 [n_bb, 1] +- ggml_tensor * pos_step, // I32 [1] +- ggml_tensor ** out_logits, // F32 [n_vocab, 1] +- ggml_tensor ** out_h_post, // F32 [n_bb, 1] +- ggml_tensor ** out_argmax) { // I32 [1] +- auto * ctx0 = gctx.ctx0; +- auto * gf = gctx.gf; +- const auto & hparams = gctx.hparams; +- const auto & cparams = gctx.cparams; +- const int n_layer = (int) gctx.n_layer; +- const int n_tokens = (int) gctx.n_tokens; +- const int n_ctx_orig = (int) gctx.n_ctx_orig; +- const int rope_type = gctx.rope_type; +- const float ext_factor = gctx.ext_factor; +- const float attn_factor = gctx.attn_factor; +- const float beta_fast = gctx.beta_fast; +- const float beta_slow = gctx.beta_slow; +- auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; +- +- const float tok_scale = sqrtf((float) target.hparams.n_embd); +- +- ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); +- cb(tok_e, "mtp_tgt_tok_embd", -1); +- +- // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). +- // Use target n_embd so Edge / non-Edge targets match the main forward. +- tok_e = ggml_scale(ctx0, tok_e, tok_scale); +- cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); +- +- ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); +- cb(inp_cat, "mtp_concat", -1); +- +- ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); +- cb(inpL, "mtp_pre_proj_out", -1); +- +- ggml_build_forward_expand(gf, inpL); +- +- ggml_tensor * cur = nullptr; +- +- for (int il = 0; il < n_layer; ++il) { +- const int64_t n_embd_head = hparams.n_embd_head_k(il); +- GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); +- +- const int64_t n_head = hparams.n_head(il); +- +- const float freq_base_l = mtp.get_rope_freq_base(cparams, il); +- const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); +- const int n_rot_l = hparams.n_rot(il); +- +- cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "attn_norm", il); +- +- ggml_tensor * freq_factors = nullptr; +- if (!hparams.is_swa(il)) { +- freq_factors = mtp.layers[il].rope_freqs; +- } +- +- ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); +- cb(Qcur, "Qcur", il); +- +- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); +- +- Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); +- cb(Qcur, "Qcur_normed", il); +- +- Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, +- ext_factor, attn_factor, beta_fast, beta_slow); +- cb(Qcur, "Qcur_pos", il); +- +- const bool read_swa = hparams.is_swa(il); +- +- const int32_t n_tgt = (int32_t) target.hparams.n_layer; +- +- // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads +- // ONE shared KV per attention type from the target — the LAST layer of that type. +- // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py +- // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} +- const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); +- +- GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); +- +- // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and +- // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the +- // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must +- // ALWAYS fetch V from the cache — not reuse the post-RoPE K tensor. +- const bool use_k_as_v = false; +- +- const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); +- const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); +- +- cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, +- hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); +- +- cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "attn_post_norm", il); +- +- ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); +- cb(attn_out, "attn_out", il); +- +- GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); +- +- cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); +- cb(cur, "ffn_norm", il); +- +- cur = gctx.build_ffn(cur, +- mtp.layers[il].ffn_up, nullptr, nullptr, +- mtp.layers[il].ffn_gate, nullptr, nullptr, +- mtp.layers[il].ffn_down, nullptr, nullptr, +- nullptr, +- LLM_FFN_GELU, LLM_FFN_PAR, il); +- cb(cur, "ffn_out", il); +- +- cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); +- cb(cur, "ffn_post_norm", il); +- +- cur = ggml_add(ctx0, cur, attn_out); +- +- if (mtp.layers[il].out_scale) { +- cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); +- cb(cur, "out_scaled", il); +- } +- +- cur = gctx.build_cvec(cur, il); +- cb(cur, "l_out", il); +- +- inpL = cur; +- } +- +- cur = inpL; +- +- cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); +- cb(cur, "result_norm", -1); +- +- ggml_tensor * h_inner = cur; +- +- ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); +- cb(backbone, "mtp_post_proj_out", -1); +- +- const int64_t n_vocab = mtp.tok_embd->ne[1]; +- const int64_t n_tokens_mtp = h_inner->ne[1]; +- +- if (mtp.hparams.use_ordered_embeddings) { +- // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full +- // [n_vocab] row then argmax — matches masked full-vocab greedy (sparse-only argmax broke server accept). +- GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); +- GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); +- const uint32_t n_c = mtp.hparams.n_centroids; +- const uint32_t top_k = mtp.hparams.centroid_top_k; +- GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); +- GGML_ASSERT(n_vocab % (int64_t) n_c == 0); +- const int64_t vsc = n_vocab / (int64_t) n_c; +- +- ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); +- cb(centroid_logits, "mtp_centroid_logits", -1); +- +- ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); +- cb(topk_idx, "mtp_centroid_topk_idx", -1); +- +- const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); +- ggml_tensor * ordering = ggml_view_2d( +- ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); +- cb(ordering, "mtp_token_ordering_view", -1); +- +- ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); +- cb(sel_ids, "mtp_selected_token_ids", -1); +- +- const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; +- ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); +- cb(flat_ids, "mtp_selected_token_ids_flat", -1); +- +- ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); +- cb(sel_emb, "mtp_selected_embd", -1); +- +- ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); +- cb(sel_logits, "mtp_selected_logits", -1); +- ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); +- +- ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); +- logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); +- cb(logits_full, "mtp_logits_masked_base", -1); +- +- ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); +- ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); +- cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); +- cb(cur, "mtp_logits_scatter_view", -1); +- cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); +- cb(cur, "mtp_logits_full", -1); +- } else { +- cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); +- cb(cur, "result_output_dense", -1); +- } +- +- if (hparams.f_final_logit_softcapping) { +- cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); +- cur = ggml_tanh(ctx0, cur); +- cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); +- } +- +- cb(cur, "result_output", -1); +- +- // Greedy argmax on-device: I32 [1] token index into the vocabulary row. +- ggml_tensor * arg = ggml_argmax(ctx0, cur); +- cb(arg, "result_argmax", -1); +- +- *out_logits = cur; +- *out_h_post = backbone; +- *out_argmax = arg; +-} +- +-llm_build_gemma4_mtp::llm_build_gemma4_mtp( +- const llama_model & target_model, +- const llama_model & mtp_model, +- const llm_graph_params & params) : +- llm_graph_context(graph_params_for_mtp(params, mtp_model)), +- target(target_model), +- mtp(mtp_model) { +- const int64_t n_bb = mtp.hparams.n_embd_out_impl; +- GGML_ASSERT(n_bb > 0); +- GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); +- +- // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 +- // (fused path) chain from the previous step's on-device argmax + post-projection, so only +- // step 0 reads token/h from the host. +- ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); +- ggml_set_input(inp_tok); +- cb(inp_tok, "mtp_inp_last_token", -1); +- +- ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); +- ggml_set_input(inp_h); +- cb(inp_h, "mtp_inp_h_prev", -1); +- +- // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into +- // ONE graph (no per-step host round-trip / GPU sync — the throughput fix). n_steps==1 is the +- // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). +- const int32_t n_steps = std::max(params.n_mtp_steps, 1); +- +- auto * inp_attn = build_attn_inp_kv_iswa(); +- +- if (n_steps <= 1) { +- { +- auto inp_wrap = std::make_unique(); +- inp_wrap->inp_last_token = inp_tok; +- inp_wrap->inp_h_prev = inp_h; +- res->add_input(std::move(inp_wrap)); +- } +- +- ggml_tensor * inp_pos = build_inp_pos(); +- +- ggml_tensor * logits = nullptr; +- ggml_tensor * h_post = nullptr; +- ggml_tensor * arg = nullptr; +- gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, +- inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); +- +- res->t_embd = h_post; +- res->t_logits = logits; +- res->t_argmax = arg; +- +- ggml_build_forward_expand(gf, arg); +- ggml_build_forward_expand(gf, h_post); +- return; +- } +- +- // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is +- // shared across steps — see gemma4_mtp_build_one_step's contract). The chain ties step k+1's +- // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. +- auto inp_wrap = std::make_unique(); +- inp_wrap->inp_last_token = inp_tok; +- inp_wrap->inp_h_prev = inp_h; +- inp_wrap->inp_pos_steps.reserve(n_steps); +- +- std::vector pos_steps; +- pos_steps.reserve(n_steps); +- for (int32_t k = 0; k < n_steps; ++k) { +- ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); +- ggml_set_input(p); +- cb(p, "mtp_inp_pos_step", k); +- pos_steps.push_back(p); +- inp_wrap->inp_pos_steps.push_back(p); +- } +- res->add_input(std::move(inp_wrap)); +- +- ggml_tensor * tok_k = inp_tok; +- ggml_tensor * h_k = inp_h; +- +- std::vector step_args; +- step_args.reserve(n_steps); +- ggml_tensor * last_logits = nullptr; +- ggml_tensor * last_h_post = nullptr; +- +- for (int32_t k = 0; k < n_steps; ++k) { +- ggml_tensor * logits_k = nullptr; +- ggml_tensor * h_post_k = nullptr; +- ggml_tensor * arg_k = nullptr; +- gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, +- tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); +- +- step_args.push_back(arg_k); +- last_logits = logits_k; +- last_h_post = h_post_k; +- +- // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the +- // target vocab — already used as a get_rows index inside build_one_step), next step's +- // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. +- tok_k = arg_k; +- h_k = h_post_k; +- } +- +- // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 +- // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend — a +- // negligible side-consumer that does NOT touch the on-device get_rows chain above. +- ggml_tensor * all_args = step_args[0]; +- for (int32_t k = 1; k < n_steps; ++k) { +- all_args = ggml_concat(ctx0, all_args, step_args[k], 0); +- } +- cb(all_args, "mtp_fused_argmax", -1); +- +- res->t_argmax = all_args; // I32 [n_steps] — the drafted token block +- res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed +- res->t_logits = last_logits; // unused on the live path (out_logits == NULL) +- +- ggml_build_forward_expand(gf, all_args); +- ggml_build_forward_expand(gf, last_h_post); +-} +- + // opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via + // llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path +-// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the +-// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 +-// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. ++// (llama_model_mapping -> this class -> load_arch_hparams/tensors). The assistant is never a ++// primary model; build_arch_graph builds the dual-context drafter graph (ctx_other required). + void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); +@@ -443,7 +73,7 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + // + // opencoti bug-858 (dual-context MTP): pre_projection is classified LLM_TENSOR_LAYER_INPUT, so the + // plain create_tensor lands it on the CPU buft. The single-context facade tolerates that (it runs on +- // the target's sched_mtp, which has the CPU backend registered), but a REAL draft context ctx_dft ++ // the target's scheduler, which has the CPU backend registered), but a REAL draft context ctx_dft + // (llama_init_from_model over the nested assistant) registers only the assistant's *offloaded* (CUDA) + // weight buffers — a CPU-resident weight then reads back uninitialized → the whole drafter forward is + // NaN → 0% accept. Force it onto the output-head (GPU-when-offloaded) buft, exactly like tok_embd +@@ -502,14 +132,12 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { + // wk/wv): each attention layer reads the TARGET's shared K/V *in place* via the standard iSWA + // build_attn (Qcur only, null k_cur/v_cur -> no store). The draft context's create_memory (A3) + // shares the target's KV cells and aliases the target's last full (n_layer-1) / SWA (n_layer-2) +-// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V — +-// which is exactly what the single-context path did explicitly via build_attn_mtp(..., il_kv). ++// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V. + // The input token embedding + backbone hidden come from the target model via ctx_other. ++// (The retired single-context facade did this explicitly via a build_attn_mtp cross-read; removed ++// in #611 once this dual-context path proved parity — see UPSTREAM_SYNC.md.) + // +-// This is DISTINCT from the single-context llm_build_gemma4_mtp (built by the gemma4 TARGET's +-// build_arch_graph, gemma4.cpp) which is kept fully intact as the fallback drafter. +-// +-// Faithful to upstream b9859 (and unlike our single-context build): dense LM head over tok_embd ++// Faithful to upstream b9859: dense LM head over tok_embd + // (no ordered-embeddings/centroid path — b9859 loads centroids but ignores them in the graph), + // no final-logit softcapping (monotonic -> argmax-invariant), and no control-vector application + // (inert without a control vector). See docs/evaluations/mtp.md. +@@ -655,8 +283,6 @@ struct llm_build_gemma4_assistant_dual : public llm_graph_context { + std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { + // opencoti bug-858 dual-context MTP (A6): build the gemma4-assistant drafter graph. This model + // is KV-less and only valid as an MTP draft context (ctx_dft) created FROM it with +- // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. The single-context +- // engine (llm_build_gemma4_mtp) is built by the gemma4 TARGET's build_arch_graph instead and +- // never reaches here; it remains the fallback drafter. ++ // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. + return std::make_unique(*this, params); + } +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +index bdf5990..464e5f5 100644 +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -132,14 +132,6 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { + } + + std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { +- // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), +- // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF +- // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ +- // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. +- if (params.gtype == LLM_GRAPH_TYPE_MTP) { +- GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); +- return std::make_unique(*this, *mtp_assistant, params); +- } + return std::make_unique(*this, params); + } + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +index deb2566..6efae4a 100644 +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -809,19 +809,10 @@ struct llama_model_gemma4 : public llama_model_base { + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; + }; + +-// opencoti F5 M6-S4 mtp +-// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. +-struct llm_build_gemma4_mtp : public llm_graph_context { +- const llama_model & target; +- const llama_model & mtp; +- +- llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); +-}; +- + // opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant — loaded INTO a gemma4 target via + // llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own +-// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the +-// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). ++// hparams/tensors; build_arch_graph builds the dual-context drafter graph (requires ++// cparams.ctx_other = the Gemma 4 target, bug-858). + struct llama_model_gemma4_assistant : public llama_model_base { + llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; +diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp +index 3a21d69..827779e 100644 +--- a/llama.cpp/src/models/qwen35.cpp ++++ b/llama.cpp/src/models/qwen35.cpp +@@ -628,7 +628,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the + // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used + // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. +- // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. + cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); + } else { + Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +@@ -700,8 +700,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr + return; + } + +- // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant +- // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver ++ // sets n_mtp_steps>1 (S3). One + // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 + // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] + // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, +diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp +index 0eb2871..af9ea1e 100644 +--- a/llama.cpp/src/models/qwen35moe.cpp ++++ b/llama.cpp/src/models/qwen35moe.cpp +@@ -680,7 +680,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the + // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used + // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. +- // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. ++ // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. + cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); + } else { + Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, +@@ -784,8 +784,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm + return; + } + +- // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant +- // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One ++ // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver ++ // sets n_mtp_steps>1 (S3). One + // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 + // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] + // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +index 34b50b1..04207b2 100644 +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -918,25 +918,21 @@ private: + + add_bos_token = llama_vocab_get_add_bos(vocab); + +- // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so +- // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file — +- // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at +- // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from +- // both native branches (separate-model draft below; NextN MTP-ctx-against-target further +- // down). Detect it via the enabled types vector, load into the target, and leave +- // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's +- // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). ++ // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4-assistant draft head is a SEPARATE GGUF (so ++ // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file, ++ // then runs as a REAL draft context (ctx_dft) whose cparams.ctx_other = ctx_tgt shares the ++ // target's KV (bug-858 dual-context, upstream b9859 shape). This is distinct from both ++ // native branches (separate-model draft below; NextN MTP-ctx-against-target further down). ++ // Detect it via the enabled types vector and load into the target first. + const bool spec_assistant = std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); + if (params_base.speculative.has_dft() && spec_assistant) { +- // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target +- // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the +- // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout +- // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. +- // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 +- // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it +- // explicitly and fail clean at boot instead of crashing mid-request. ++ // opencoti F5 M6-S4 mtp (#485/#611): with --parallel >1 WITHOUT --kv-unified the KV is ++ // per-stream-split (patch 0031); the assistant's shared-KV aliasing (mem_other) assumes the ++ // unified per-layer tensor layout, so per-stream splits are not supported. The unified ++ // cache IS handled (proven on the facade at --parallel 2 --kv-unified — the PolyKV config, ++ // which forces --kv-unified). Require it explicitly and fail clean at boot. + if (params_base.n_parallel > 1 && !params_base.kv_unified) { + SRV_ERR("%s", + "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " +@@ -965,21 +961,19 @@ private: + return false; + } + +- // opencoti bug-858 dual-context MTP (A5): optionally create a REAL draft context FROM the +- // nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 ++ // opencoti bug-858 dual-context MTP (A5, default since #611): create a REAL draft context ++ // FROM the nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 + // dual-context). The assistant is KV-less: its create_memory (A3) shares the target's KV + // cells + aliases the last full/SWA layer, and its graph (A6) reads the target's tok_embd + // + shared K/V via ctx_other; the shared-KV draft_mtp driver (is_mem_shared) then runs it. +- // Gated by OPENCOTI_MTP_DUAL_CTX so the default keeps the proven single-context in-target +- // path (ctx_dft == nullptr, llama_decode_mtp) byte-identical. +- const char * mtp_dual_env = getenv("OPENCOTI_MTP_DUAL_CTX"); +- const bool mtp_dual_ctx = mtp_dual_env && mtp_dual_env[0] && mtp_dual_env[0] != '0'; +- if (mtp_dual_ctx) { ++ // This is the ONLY assistant execution path — the single-context in-target facade ++ // (llama_decode_mtp) was removed in #611 after the port proved accept parity. ++ { + // llama_init_from_model wants a mutable model*; the nested assistant is owned mutably + // by the target (unique_ptr), so the const_cast is safe. + llama_model * assistant = const_cast(llama_model_get_mtp_assistant(model_tgt)); + if (assistant == nullptr) { +- SRV_ERR("%s", "MTP dual-context requested but the assistant is not loaded into the target\n"); ++ SRV_ERR("%s", "MTP assistant is not loaded into the target\n"); + return false; + } + +@@ -1002,11 +996,6 @@ private: + params_base.speculative.draft.ctx_tgt = ctx_tgt; + params_base.speculative.draft.ctx_dft = ctx_dft.get(); + SRV_INF("%s", "MTP assistant dual-context draft created (ctx_other=target, shared KV)\n"); +- } else { +- // No separate draft context: the assistant decodes on ctx_tgt via llama_decode_mtp. +- params_base.speculative.draft.ctx_tgt = ctx_tgt; +- params_base.speculative.draft.ctx_dft = nullptr; +- SRV_INF("%s", "MTP assistant loaded into target (single-context)\n"); + } + } else if (params_base.speculative.has_dft()) { + // TODO speculative: move to common/speculative.cpp?