diff --git "a/patches/0079-gemma-assistant-mtp.patch" "b/patches/0079-gemma-assistant-mtp.patch" new file mode 100644--- /dev/null +++ "b/patches/0079-gemma-assistant-mtp.patch" @@ -0,0 +1,3013 @@ +From: opencoti +Subject: [PATCH 0079] Gemma 4 assistant-drafter MTP — restore onto 0.10.3, coexist with native NextN (#423) + +Restores the Gemma-4 A4B speculative MTP assistant drafter (the gemma4-assistant sub-model, +loaded INTO a gemma4 target via llama_model_load_mtp_from_file) on the llamafile 0.10.3 base, +as a first-class `--spec-type draft-assistant` that COEXISTS with the upstream-native NextN +MTP (`--spec-type draft-mtp`). Distinct enums (COMMON_SPECULATIVE_TYPE_MTP vs +COMMON_SPECULATIVE_TYPE_DRAFT_MTP) and graph types (LLM_GRAPH_TYPE_MTP vs +LLM_GRAPH_TYPE_DECODER_MTP) — no collision. + +The drafter cross-reads the target KV read-only at decode time (build_attn_mtp) and decodes on +ctx_tgt via llama_decode_mtp; no second llama_context / KV cache. It rides the 0.10.3 seq-aware +speculative framework (process/draft/accept/need_embd hooks replace the old server h_idx/seq_id +plumbing, so it composes with PolyKV / multi-seq for free) and with the turbo / rolling-KV / DCA +features below it in the stack. + +Key 0.10.3 adaptations vs the original 0074 capture: +- per-model class (llama_model_gemma4_assistant) via llama_model_mapping, not the obsolete + 0.10.1 giant-switch loader; +- llm_graph_input_attn_kv_iswa::set_input DECOUPLES the kq_mask setter from the self_k_idxs + buffer guard. 0.10.3 nested them, which would silently starve the read-only MTP draft graph's + live mask when its pruned KV-write idxs (null buffer) skip the whole base block — a silent + re-introduction of bug-404. Each idxs/mask/rot setter now guards on its own buffer; + byte-identical for every non-MTP graph. +- server loader gains a leading `has_dft() && assistant` branch that loads the assistant into + the target and leaves model_dft/ctx_dft null (null-safe across every ctx_dft checkpoint path, + which all early-return on a null context). + +opencoti upstream-alignment (#476 S1; see docs/protocols/UPSTREAM_SYNC.md): this patch now carries +the upstream llama.cpp #23398/#24282 DATA-PLANE identity for the assistant drafter — arch string +`gemma4-assistant` (hyphen), tensors `nextn.pre_projection`/`nextn.post_projection` + +`masked_embd_centroids`/`masked_embd_ordering`, and the generic `embedding_length_out` / +`nextn_predict_layers` KV — plus the matching internal C++ symbols (LLM_TENSOR_NEXTN_PROJ_*, +LLM_TENSOR_MASKED_EMBD_*, hparams n_embd_out_impl). The CONTROL-PLANE is deliberately KEPT +opencoti-native (single-context build_attn_mtp cross-read — NOT upstream's ctx_other second context, +which assumes vanilla single-owner unquantized KV and breaks PolyKV/rolling-KV/turbo/DCA; ordered +masked-embedder head; tok_embd→OUTPUT buft; fused N-step; `--spec-type draft-assistant`). The rename +is semantically INERT — proven by real_frac=0 logit-equivalence on the re-formatted A4B drafter. + +opencoti upstream-alignment (#476 S1; see docs/protocols/UPSTREAM_SYNC.md): this patch now carries +the upstream llama.cpp #23398/#24282 DATA-PLANE identity for the assistant drafter — arch string +`gemma4-assistant` (hyphen), tensors `nextn.pre_projection`/`nextn.post_projection` + +`masked_embd_centroids`/`masked_embd_ordering`, and the generic `embedding_length_out` / +`nextn_predict_layers` KV — plus the matching internal C++ symbols (LLM_TENSOR_NEXTN_PROJ_*, +LLM_TENSOR_MASKED_EMBD_*, hparams n_embd_out_impl). The CONTROL-PLANE is deliberately KEPT +opencoti-native (single-context build_attn_mtp cross-read — NOT upstream's ctx_other second context, +which assumes vanilla single-owner unquantized KV and breaks PolyKV/rolling-KV/turbo/DCA; ordered +masked-embedder head; tok_embd→OUTPUT buft; fused N-step; `--spec-type draft-assistant`). The rename +is semantically INERT — proven by real_frac=0 logit-equivalence on the re-formatted A4B drafter. + +diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk +--- a/llama.cpp/BUILD.mk ++++ b/llama.cpp/BUILD.mk +@@ -109,6 +109,7 @@ + llama.cpp/src/models/gemma3.cpp \ + llama.cpp/src/models/gemma3n.cpp \ + llama.cpp/src/models/gemma4.cpp \ ++ llama.cpp/src/models/gemma4-assistant.cpp \ + llama.cpp/src/models/glm-dsa.cpp \ + llama.cpp/src/models/glm4-moe.cpp \ + llama.cpp/src/models/glm4.cpp \ +diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp +--- a/llama.cpp/common/arg.cpp ++++ b/llama.cpp/common/arg.cpp +@@ -3816,6 +3816,13 @@ + params.speculative.draft.mparams.path = value; + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); ++ add_opt(common_arg( // opencoti F5 M6-S4 mtp — Gemma 4 assistant drafter ++ {"--mtp-head"}, "FNAME", ++ "alias for Gemma 4 MTP: path to gemma4_assistant GGUF (loaded into the target; use with --spec-type draft-assistant)", ++ [](common_params & params, const std::string & value) { ++ params.speculative.draft.mparams.path = value; ++ } ++ ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_MTP_HEAD")); + add_opt(common_arg( + {"--spec-type"}, common_speculative_all_types_str(), + string_format("comma-separated list of types of speculative decoding to use (default: %s)\n", +@@ -3826,6 +3833,16 @@ + params.speculative.types.insert(params.speculative.types.end(), types.begin(), types.end()); + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_TYPE")); ++ add_opt(common_arg( // opencoti F5 M6-S4 mtp ++ {"--draft-block-size"}, "N", ++ string_format("MTP draft block size B (drafts B-1 tokens per round; default: %d)", params.speculative.draft.draft_block_size), ++ [](common_params & params, int value) { ++ if (value < 2 || value > 32) { ++ throw std::invalid_argument("draft block size must be between 2 and 32"); ++ } ++ params.speculative.draft.draft_block_size = value; ++ } ++ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_SPECULATIVE}).set_env("LLAMA_ARG_DRAFT_BLOCK_SIZE")); + add_opt(common_arg( + {"--spec-ngram-mod-n-min"}, "N", + string_format("minimum number of ngram tokens to use for ngram-based speculative decoding (default: %d)", params.speculative.ngram_mod.n_min), +diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h +--- a/llama.cpp/common/common.h ++++ b/llama.cpp/common/common.h +@@ -166,6 +166,7 @@ + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values + COMMON_SPECULATIVE_TYPE_NGRAM_MOD, + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache ++ COMMON_SPECULATIVE_TYPE_MTP, // opencoti F5 M6-S4 — Gemma 4 MTP assistant drafter + COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type + }; + +@@ -302,6 +303,10 @@ + int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding + int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding + ++ // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): draft block size B produces ++ // B-1 draft tokens per round. Default 3 (= 2 chained MTP draft steps). ++ int32_t draft_block_size = 3; ++ + float p_split = 0.1f; // speculative decoding split probability + float p_min = 0.0f; // minimum speculative decoding probability (greedy) + +diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp +--- a/llama.cpp/common/speculative.cpp ++++ b/llama.cpp/common/speculative.cpp +@@ -16,6 +16,11 @@ + #include + #include + #include ++#include // opencoti F5 M6-S4 mtp: std::sqrt (compute_h_l2) ++#include // opencoti F5 M6-S4 mtp: FILE/fopen/fputs (mtp_acc_tracer) ++#include // opencoti F5 M6-S4 mtp: std::getenv/std::atoi ++#include // opencoti F5 M6-S4 mtp: std::mutex (mtp_acc_tracer) ++#include // opencoti F5 M6-S4 mtp: std::ostringstream (trace_emit_*) + + #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 + #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 +@@ -29,7 +34,8 @@ + {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, + {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, + {"ngram-mod", COMMON_SPECULATIVE_TYPE_NGRAM_MOD}, +- {"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE} ++ {"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}, ++ {"draft-assistant", COMMON_SPECULATIVE_TYPE_MTP} // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter (distinct from native draft-mtp/NextN) + }; + + static std::string common_speculative_get_devices_str(const std::vector & devices) { +@@ -52,6 +58,63 @@ + const common_params_speculative & p = common_params_speculative{}) : type(t), params(p) {} + }; + ++// opencoti F5 M6-S4 mtp: target must be gemma4, assistant must be gemma4_assistant (loaded into target). ++// The fork uses a fork-only llama_model_arch_str() API; OURS reads the "general.architecture" GGUF key ++// via the existing public llama_model_meta_val_str — that key is exactly what the loader used to select ++// the arch enum, so it returns "gemma4" / "gemma4-assistant" canonically. (No new C API — P3 is host-only.) ++static bool common_speculative_mtp_arch_ok(const llama_model * model_tgt, const llama_model * model_dft) { ++ char arch_tgt[64] = {0}; ++ char arch_dft[64] = {0}; ++ llama_model_meta_val_str(model_tgt, "general.architecture", arch_tgt, sizeof(arch_tgt)); ++ llama_model_meta_val_str(model_dft, "general.architecture", arch_dft, sizeof(arch_dft)); ++ return std::strcmp(arch_tgt, "gemma4") == 0 ++ && std::strcmp(arch_dft, "gemma4-assistant") == 0; ++} ++ ++// opencoti F5 M6-S4 mtp: MTP-specific vocab compatibility. The assistant predicts the next token id ++// from the SAME SentencePiece vocab; stop/chat special tokens are owned by the target. So we require ++// identical vocab_type, size (within tolerance) and per-token text equality, and intentionally skip ++// bos/eos id + add_bos/add_eos checks (target may be chat-tuned eos==106, draft eos==1). ++static bool common_speculative_are_compatible_mtp( ++ const llama_model * model_tgt, ++ const llama_model * model_dft) { ++ const llama_vocab * vocab_tgt = llama_model_get_vocab(model_tgt); ++ const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); ++ ++ // opencoti F5 M6-S4 mtp: the gemma4_assistant draft head carries NO tokenizer (vocab type NONE); ++ // it shares the target's vocab, and load-time llama_model_load_mtp_from_file already verified ++ // n_tokens-equality. A NONE vocab has no token text to compare and is compatible by construction — ++ // accept here rather than failing the type/text checks below (which assume the draft has a real ++ // vocab; NONE != the target's real type would otherwise reject the assistant outright). ++ if (llama_vocab_type(vocab_dft) == LLAMA_VOCAB_TYPE_NONE) { ++ return true; ++ } ++ ++ if (llama_vocab_type(vocab_tgt) != llama_vocab_type(vocab_dft)) { ++ return false; ++ } ++ ++ const int n_vocab_tgt = llama_vocab_n_tokens(vocab_tgt); ++ const int n_vocab_dft = llama_vocab_n_tokens(vocab_dft); ++ const int vocab_diff = n_vocab_tgt > n_vocab_dft ++ ? n_vocab_tgt - n_vocab_dft ++ : n_vocab_dft - n_vocab_tgt; ++ ++ if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { ++ return false; ++ } ++ ++ for (int i = SPEC_VOCAB_CHECK_START_TOKEN_ID; i < std::min(n_vocab_tgt, n_vocab_dft); ++i) { ++ const char * t_tgt = llama_vocab_get_text(vocab_tgt, i); ++ const char * t_dft = llama_vocab_get_text(vocab_dft, i); ++ if (std::strcmp(t_tgt, t_dft) != 0) { ++ return false; ++ } ++ } ++ ++ return true; ++} ++ + static bool common_speculative_are_compatible( + const llama_model * model_tgt, + const llama_model * model_dft) { +@@ -1206,6 +1269,311 @@ + } + }; + ++// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. ++// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, ++// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV ++// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens ++// of a block in ONE facade call (no per-step sampler loop). ++// ++// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), ++// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never ++// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() ++// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) ++// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). ++ ++// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. ++// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. ++namespace { ++struct mtp_acc_tracer { ++ bool enabled = false; ++ FILE * fp = nullptr; ++ std::mutex mu; ++ ++ mtp_acc_tracer() { ++ const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); ++ if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { ++ return; ++ } ++ fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); ++ enabled = (fp != nullptr); ++ } ++ ++ ~mtp_acc_tracer() { ++ if (fp && fp != stderr) { ++ std::fclose(fp); ++ } ++ } ++ ++ void writeln(const std::string & line) { ++ if (!enabled) { ++ return; ++ } ++ std::lock_guard lk(mu); ++ std::fputs(line.c_str(), fp); ++ std::fputc('\n', fp); ++ std::fflush(fp); ++ } ++}; ++ ++mtp_acc_tracer & mtp_tracer() { ++ static mtp_acc_tracer t; ++ return t; ++} ++} // namespace ++ ++struct common_speculative_impl_draft_assistant : public common_speculative_impl { ++ common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) ++ ++ llama_context * ctx_tgt = nullptr; ++ ++ int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes ++ int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row ++ ++ // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows ++ // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds ++ // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h ++ // scheme so multi-seq (PolyKV) works without per-seq server plumbing. ++ std::vector> pending_h; // [n_seq][n_embd_backbone] ++ std::vector> verify_h; // [n_seq][n_rows * n_embd_backbone] ++ std::vector verify_h_rows; ++ std::vector i_batch_beg; ++ std::vector i_batch_end; ++ ++ // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head ++ // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain ++ // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. ++ int skip_streak_threshold = 0; ++ std::vector zero_accept_streak; ++ std::vector skip_last_draft; ++ std::vector prev_n_acc_at_draft; ++ ++ int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change ++ ++ static float compute_h_l2(const float * h, int32_t n) { ++ if (!h || n <= 0) { ++ return 0.0f; ++ } ++ double s = 0.0; ++ for (int32_t i = 0; i < n; ++i) { ++ s += (double) h[i] * (double) h[i]; ++ } ++ return (float) std::sqrt(s); ++ } ++ ++ common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) ++ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) ++ , params(params.draft) ++ { ++ ctx_tgt = this->params.ctx_tgt; ++ GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); ++ ++ const llama_model * model_tgt = llama_get_model(ctx_tgt); ++ n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); ++ n_embd_out = llama_model_n_embd_out(model_tgt); ++ ++ LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); ++ LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, ++ this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); ++ ++ // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. ++ llama_set_embeddings(ctx_tgt, true); ++ ++ if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { ++ const int v = std::atoi(e); ++ if (v >= 1 && v <= 32) { ++ skip_streak_threshold = v; ++ } ++ } ++ ++ pending_h.assign(n_seq, std::vector((size_t) std::max(1, n_embd_backbone), 0.0f)); ++ verify_h.assign(n_seq, {}); ++ verify_h_rows.assign(n_seq, 0); ++ i_batch_beg.assign(n_seq, -1); ++ i_batch_end.assign(n_seq, -1); ++ zero_accept_streak.assign(n_seq, 0); ++ skip_last_draft.assign(n_seq, 0); ++ prev_n_acc_at_draft.assign(n_seq, 0); ++ } ++ ++ ~common_speculative_impl_draft_assistant() override = default; ++ ++ void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { ++ llama_set_embeddings(ctx_tgt, true); ++ if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { ++ skip_last_draft[seq_id] = 0; ++ zero_accept_streak[seq_id] = 0; ++ } ++ } ++ ++ // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() ++ // can select the last-accepted token's hidden state. The spec framework calls process() after the ++ // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which ++ // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. ++ bool process(const llama_batch & batch_in) override { ++ if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { ++ return true; ++ } ++ if (n_embd_backbone <= 0) { ++ return true; ++ } ++ ++ const int32_t n_tokens = batch_in.n_tokens; ++ std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); ++ std::fill(i_batch_end.begin(), i_batch_end.end(), -1); ++ ++ for (int k = 0; k < n_tokens; ++k) { ++ const llama_seq_id s = batch_in.seq_id[k][0]; ++ if (s < 0 || s >= (llama_seq_id) n_seq) { ++ continue; ++ } ++ i_batch_end[s] = k; ++ if (i_batch_beg[s] < 0) { ++ i_batch_beg[s] = k; ++ } ++ } ++ ++ const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); ++ const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); ++ ++ for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { ++ if (i_batch_beg[s] < 0) { ++ continue; ++ } ++ const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; ++ verify_h_rows[s] = n_rows; ++ verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); ++ ++ for (int32_t i = 0; i < n_rows; ++i) { ++ const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); ++ if (h) { ++ std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, ++ (size_t) n_copy * sizeof(float)); ++ } ++ } ++ // default pending_h to the last row (correct when all drafts accepted / on prefill) ++ std::memcpy(pending_h[s].data(), ++ verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); ++ } ++ ++ return true; ++ } ++ ++ void draft(common_speculative_draft_params_vec & dparams) override { ++ if (n_embd_backbone <= 0) { ++ return; ++ } ++ ++ const int32_t block = params.draft_block_size; ++ const int32_t n_steps_raw = block > 1 ? block - 1 : 0; ++ ++ for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { ++ auto & dp = dparams[seq_id]; ++ if (!dp.drafting) { ++ continue; ++ } ++ ++ // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, ++ // the previous batch produced 0 accepted drafts. ++ if (skip_last_draft[seq_id]) { ++ skip_last_draft[seq_id] = 0; ++ } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { ++ zero_accept_streak[seq_id]++; ++ } else { ++ zero_accept_streak[seq_id] = 0; ++ } ++ if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { ++ zero_accept_streak[seq_id] = 0; ++ skip_last_draft[seq_id] = 1; ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ continue; // empty result -> server falls back to single-token verify ++ } ++ ++ int32_t n_steps = n_steps_raw; ++ if (dp.n_max > 0) { ++ n_steps = std::min(n_steps, dp.n_max); ++ } ++ n_steps = std::min(n_steps, params.n_max); ++ if (n_steps <= 0) { ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ continue; ++ } ++ ++ float * h_prev = pending_h[seq_id].data(); ++ ++ llama_memory_t mem = llama_get_memory(ctx_tgt); ++ llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; ++ if (attn_pos < 0) { ++ attn_pos = 0; ++ } ++ ++ std::vector out((size_t) n_steps, 0); ++ const int32_t rc = llama_decode_mtp( ++ ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, ++ /*out_drafts =*/ out.data(), ++ /*out_logits =*/ nullptr, ++ /*out_h_prev_last=*/ nullptr); ++ ++ prev_n_acc_at_draft[seq_id] = n_acc_drafts; ++ ++ if (rc != 0) { ++ LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); ++ continue; ++ } ++ ++ auto & result = *dp.result; ++ for (int32_t i = 0; i < n_steps; ++i) { ++ result.push_back(out[i]); ++ } ++ ++ if (mtp_tracer().enabled) { ++ std::ostringstream oss; ++ oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter ++ << ",\"seq_id\":" << (int) seq_id ++ << ",\"id_last\":" << (int) dp.id_last ++ << ",\"attn_pos\":" << (int) attn_pos ++ << ",\"n_steps\":" << n_steps ++ << ",\"h_l2\":" << std::fixed << std::setprecision(4) ++ << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; ++ for (int32_t i = 0; i < n_steps; ++i) { ++ if (i) oss << ','; ++ oss << (int) out[i]; ++ } ++ oss << "]}"; ++ mtp_tracer().writeln(oss.str()); ++ } ++ } ++ } ++ ++ void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { ++ if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { ++ return; ++ } ++ const int32_t n_rows = verify_h_rows[seq_id]; ++ if (n_rows <= 0 || n_embd_backbone <= 0) { ++ return; ++ } ++ // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the ++ // last-accepted token's hidden state (clamped to the harvested rows). ++ const int32_t i_h = std::min(n_accepted, n_rows - 1); ++ std::memcpy(pending_h[seq_id].data(), ++ verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, ++ (size_t) n_embd_backbone * sizeof(float)); ++ ++ if (mtp_tracer().enabled) { ++ std::ostringstream oss; ++ oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter ++ << ",\"seq_id\":" << (int) seq_id ++ << ",\"n_accepted\":" << (int) n_accepted << "}"; ++ mtp_tracer().writeln(oss.str()); ++ ++trace_iter; ++ } ++ } ++ ++ bool need_embd() const override { ++ return true; // MTP reads POST-norm output embeddings (h_prev) ++ } ++}; ++ + struct common_speculative { + common_speculative_draft_params_vec dparams; + +@@ -1278,6 +1646,7 @@ + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; + case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: return "ngram-mod"; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: return "ngram-cache"; ++ case COMMON_SPECULATIVE_TYPE_MTP: return "draft-assistant"; // opencoti F5 M6-S4 mtp + default: return "unknown"; + } + } +@@ -1330,6 +1699,9 @@ + bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); + bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 + bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; ++ // opencoti F5 M6-S4 mtp: our Gemma-4 assistant drafter (distinct from native NextN _DRAFT_MTP). ++ // Loaded INTO the target (no ctx_dft); validated against the in-target assistant below. ++ bool has_mtp_assistant = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_MTP)) != 0; + + bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); + bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); +@@ -1338,7 +1710,7 @@ + bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); + + // when adding a new type - update here the logic above +- static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); ++ static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); // opencoti F5 M6-S4 mtp: +_MTP (Gemma-4 assistant drafter) + + // this list here defines the priority of the speculators + // the one with highest priority are listed first +@@ -1364,7 +1736,11 @@ + LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__); + has_draft_simple = false; + } +- } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) { ++ } else if (has_draft_model_path && !has_mtp && !has_mtp_assistant && !has_draft_eagle3) { ++ // opencoti F5 M6-S4 mtp: do NOT auto-fallback to draft-simple when our Gemma-4 assistant ++ // is the draft type. The assistant loads INTO the target (ctx_dft == nullptr by design), ++ // so a draft-simple impl would dereference the null draft context and crash. The assistant ++ // path is set up by the has_mtp_assistant branch below. + LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__); + has_draft_simple = true; + } +@@ -1378,6 +1754,20 @@ + if (has_mtp) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); + } ++ if (has_mtp_assistant) { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter (validate in-target assistant) ++ const llama_model * model_tgt = params.draft.ctx_tgt ? llama_get_model(params.draft.ctx_tgt) : nullptr; ++ const llama_model * model_mtp = model_tgt ? llama_model_get_mtp_assistant(model_tgt) : nullptr; ++ if (!model_mtp) { ++ LOG_WRN("%s: 'draft-assistant' requires the gemma4_assistant GGUF loaded into the target " ++ "(--spec-type draft-assistant with --mtp-head/--model-draft); disabling\n", __func__); ++ } else if (!common_speculative_mtp_arch_ok(model_tgt, model_mtp)) { ++ LOG_WRN("%s: 'draft-assistant' requires target arch gemma4 + assistant arch gemma4_assistant; disabling\n", __func__); ++ } else if (!common_speculative_are_compatible_mtp(model_tgt, model_mtp)) { ++ LOG_WRN("%s: 'draft-assistant' assistant failed vocab compatibility; disabling\n", __func__); ++ } else { ++ configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_MTP, params)); ++ } ++ } + } + + std::vector> impls = {}; +@@ -1398,6 +1788,10 @@ + impls.push_back(std::make_unique(config.params, n_seq)); + break; + } ++ case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter ++ impls.push_back(std::make_unique(config.params, n_seq)); ++ break; ++ } + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { + common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); + +@@ -1460,6 +1854,12 @@ + return result; + } + ++// opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / ++// set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative ++// framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ ++// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on ++// common_speculative_impl_draft_assistant above). ++ + void common_speculative_free(common_speculative * spec) { + if (spec == nullptr) { + return; +diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h +--- a/llama.cpp/include/llama.h ++++ b/llama.cpp/include/llama.h +@@ -540,6 +540,21 @@ + size_t n_paths, + struct llama_model_params params); + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP: load gemma4_assistant GGUF into a gemma4 target (call after ++ // llama_model_load_from_file, before llama_init_from_model). Returns 0 on success. ++ LLAMA_API int llama_model_load_mtp_from_file( ++ struct llama_model * model, ++ const char * path_mtp, ++ struct llama_model_params params); ++ ++ LLAMA_API const struct llama_model * llama_model_get_mtp_assistant(const struct llama_model * model); ++ ++ LLAMA_API bool llama_model_has_mtp_assistant(const struct llama_model * model); ++ ++ // Backbone hidden size for MTP input (0 if no MTP assistant is loaded). ++ LLAMA_API uint32_t llama_model_mtp_n_embd_backbone(const struct llama_model * model); ++ + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); +@@ -1010,6 +1025,22 @@ + struct llama_context * ctx, + struct llama_batch batch); + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, ++ // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). ++ // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) ++ // receive per-step logits and the final backbone hidden. Returns 0 on success. ++ LLAMA_API int32_t llama_decode_mtp( ++ struct llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) +diff --git a/llama.cpp/src/llama-arch.cpp b/llama.cpp/src/llama-arch.cpp +--- a/llama.cpp/src/llama-arch.cpp ++++ b/llama.cpp/src/llama-arch.cpp +@@ -57,6 +57,7 @@ + { LLM_ARCH_GEMMA3, "gemma3" }, + { LLM_ARCH_GEMMA3N, "gemma3n" }, + { LLM_ARCH_GEMMA4, "gemma4" }, ++ { LLM_ARCH_GEMMA4_ASSISTANT, "gemma4-assistant" }, // opencoti F5 M6-S4 mtp — upstream-aligned arch string (#23398, hyphen) + { LLM_ARCH_GEMMA_EMBEDDING, "gemma-embedding" }, + { LLM_ARCH_STARCODER2, "starcoder2" }, + { LLM_ARCH_MAMBA, "mamba" }, +@@ -293,6 +294,14 @@ + { LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" }, + { LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" }, + ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP, opencoti-private additive KV (upstream #23398/#24282 ++ // has no equivalent; backbone width moved to the generic %s.embedding_length_out). %s = "gemma4-assistant". ++ { LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, "%s.n_centroids" }, ++ { LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, "%s.centroid_top_k" }, ++ { LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, "%s.attention.k_eq_v" }, ++ { LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, "%s.use_ordered_embeddings" }, ++ { LLM_KV_GEMMA4_ASSISTANT_REQUIRES_TARGET_ARCH, "%s.requires_target_arch" }, ++ + { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" }, + { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" }, + { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" }, +@@ -452,6 +461,11 @@ + { LLM_TENSOR_NEXTN_HNORM, "blk.%d.nextn.hnorm" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "blk.%d.nextn.shared_head_head" }, + { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "blk.%d.nextn.shared_head_norm" }, ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant tensors ++ { LLM_TENSOR_NEXTN_PROJ_PRE, "nextn.pre_projection" }, ++ { LLM_TENSOR_NEXTN_PROJ_POST, "nextn.post_projection" }, ++ { LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" }, ++ { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, + { LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" }, + { LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" }, + { LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" }, +@@ -767,6 +781,11 @@ + {LLM_TENSOR_NEXTN_HNORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant tensors ++ {LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_INPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_MASKED_EMBD_CENTROIDS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_MASKED_EMBD_ORDERING, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, + // Nemotron 3 Super + // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU + {LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, +diff --git a/llama.cpp/src/llama-arch.h b/llama.cpp/src/llama-arch.h +--- a/llama.cpp/src/llama-arch.h ++++ b/llama.cpp/src/llama-arch.h +@@ -61,6 +61,7 @@ + LLM_ARCH_GEMMA3, + LLM_ARCH_GEMMA3N, + LLM_ARCH_GEMMA4, ++ LLM_ARCH_GEMMA4_ASSISTANT, // opencoti F5 M6-S4 mtp + LLM_ARCH_GEMMA_EMBEDDING, + LLM_ARCH_STARCODER2, + LLM_ARCH_MAMBA, +@@ -345,6 +346,16 @@ + LLM_KV_DENSE_2_FEAT_OUT, + LLM_KV_DENSE_3_FEAT_IN, + LLM_KV_DENSE_3_FEAT_OUT, ++ ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP, opencoti-private additive KV (no upstream KV ++ // equivalent; upstream #24282 detects the ordered/masked-embedder head by tensor presence, not KV ++ // flags). Backbone width is NOT here: it reuses the generic LLM_KV_EMBEDDING_LENGTH_OUT, matching ++ // upstream. The "%s." prefix auto-follows the arch string (now "gemma4-assistant"). ++ LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, ++ LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, ++ LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, ++ LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, ++ LLM_KV_GEMMA4_ASSISTANT_REQUIRES_TARGET_ARCH, + }; + + enum llm_tensor { +@@ -556,6 +567,15 @@ + LLM_TENSOR_NEXTN_HNORM, + LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, + LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, ++ ++ // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 ++ // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile ++ // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our ++ // single-context build_attn_mtp cross-read — NOT upstream's ctx_other. See plan. ++ LLM_TENSOR_NEXTN_PROJ_PRE, ++ LLM_TENSOR_NEXTN_PROJ_POST, ++ LLM_TENSOR_MASKED_EMBD_CENTROIDS, ++ LLM_TENSOR_MASKED_EMBD_ORDERING, + }; + + enum llm_tensor_layer { +diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp +--- a/llama.cpp/src/llama-context.cpp ++++ b/llama.cpp/src/llama-context.cpp +@@ -9,6 +9,7 @@ + #include "llama-memory.h" + #include "llama-mmap.h" + #include "llama-model.h" ++#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp + #include "llama-ext.h" + #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard + #include "llama.h" +@@ -432,6 +433,17 @@ + } + + llama_context::~llama_context() { ++ // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down ++ // sched_mtp and the backends it touches. ++ if (mtp_worker.joinable()) { ++ { ++ std::lock_guard lk(mtp_mu); ++ mtp_worker_stop.store(true, std::memory_order_release); ++ } ++ mtp_cv_request.notify_all(); ++ mtp_worker.join(); ++ } ++ + if (!model.hparams.no_alloc) { + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + ggml_backend_t backend = backend_ptrs[i]; +@@ -1313,6 +1325,208 @@ + return res; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++bool llama_context::ensure_sched_mtp() { ++ // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count ++ // (mtp_fused_steps). A fused N-step graph has ~N× the nodes of the single-step graph and a ++ // distinct topology, so when a larger block is requested we drop and rebuild the reserve. ++ const int32_t need = std::max(mtp_fused_steps, 1); ++ if (sched_mtp && mtp_reserved_steps >= need) { ++ return true; ++ } ++ if (!model.mtp_assistant) { ++ return false; ++ } ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ ++ const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); ++ const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; ++ ++ gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); ++ sched_mtp.reset(ggml_backend_sched_new( ++ backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), ++ max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); ++ if (!sched_mtp) { ++ LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. ++ // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), ++ // so a single reserve covers all subsequent decode_mtp calls. ++ { ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ llama_memory_context_ptr mctx = memory->init_full(); ++ if (!mctx) { ++ LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(0); ++ data->seq_idx[0] = 0; ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = 0; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ const uint32_t save_n_outputs = n_outputs; ++ n_outputs = 1; ++ ++ auto * res = gf_res_prev_mtp.get(); ++ const auto gparams = graph_params_mtp(res, ub, mctx.get()); ++ res->reset(); ++ ggml_backend_sched_reset(sched_mtp.get()); ++ ++ auto * gf = model.build_graph(gparams); ++ n_outputs = save_n_outputs; ++ ++ if (!gf) { ++ LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { ++ LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); ++ sched_mtp.reset(); ++ gf_res_prev_mtp.reset(); ++ return false; ++ } ++ // Discard the reserve graph cache so the first real call rebuilds with proper inputs. ++ res->reset(); ++ ggml_backend_sched_reset(sched_mtp.get()); ++ } ++ ++ // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in ++ // decode_mtp_async (only when the opt-in async path is actually taken), NOT here — so the ++ // default in-thread sync path creates no extra thread and is runtime-identical to P3. ++ ++ mtp_reserved_steps = need; // reserve now covers `need` fused steps ++ return true; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++llm_graph_result * llama_context::process_ubatch_mtp( ++ const llama_ubatch & ubatch, ++ llama_memory_context_i * mctx, ++ ggml_status & ret) { ++ GGML_ASSERT(sched_mtp && gf_res_prev_mtp); ++ ++ auto * res = gf_res_prev_mtp.get(); ++ auto * gf = res->get_gf(); ++ ++ // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it ++ // explicitly here in case a concurrent target decode mutates it. The MTP graph ++ // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. ++ llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); ++ gparams.n_outputs = 1; ++ gparams.sched = sched_mtp.get(); ++ ++ if (!graph_reuse_disable && res->can_reuse(gparams)) { ++ // sched_mtp does not use pipeline parallelism (created with pipeline=false), ++ // so no synchronize is needed before set_inputs. ++ } else { ++ res->reset(); ++ ++ ggml_backend_sched_reset(sched_mtp.get()); ++ ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); ++ ++ gf = model.build_graph(gparams); ++ if (!gf) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); ++ ret = GGML_STATUS_FAILED; ++ return nullptr; ++ } ++ if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { ++ LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); ++ ret = GGML_STATUS_ALLOC_FAILED; ++ return nullptr; ++ } ++ } ++ ++ res->set_inputs(&ubatch); ++ ++ const auto status = graph_compute_mtp(res->get_gf()); ++ ++ if (status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); ++ ret = status; ++ return nullptr; ++ } ++ ++ ret = GGML_STATUS_SUCCESS; ++ return res; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { ++ // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s ++ // threadpool/n_threads dance only for the CPU backend; the MTP head is small and ++ // thread saturation matters less, but consistency with graph_compute() avoids ++ // surprising backend reconfigurations between target and MTP runs. ++ // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker ++ // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. ++ const int n_threads = cparams.n_threads; ++ ggml_threadpool_t tp = threadpool; ++ ++ if (backend_cpu != nullptr) { ++ auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); ++ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ++ ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); ++ if (set_threadpool_fn) { ++ set_threadpool_fn(backend_cpu, tp); ++ } ++ } ++ for (const auto & set_n_threads_fn : set_n_threads_fns) { ++ set_n_threads_fn.second(set_n_threads_fn.first, n_threads); ++ } ++ ++ auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); ++ if (status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", ++ __func__, status); ++ } ++ return status; ++} ++ + llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { + if (mctx && !mctx->apply()) { + LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); +@@ -1773,6 +1987,14 @@ + + bool did_optimize = false; + ++ // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its ++ // KV below (memory_update's pending shifts/copies, then init_batch's slot apply — the S1 ++ // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this ++ // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in ++ // the current sync-facade integration (nothing is in flight here) and a single pointer ++ // check when no MTP assistant is loaded. ++ mtp_drain_before_mutate(); ++ + // handle any pending shifts/copies + memory_update(false); + +@@ -2347,6 +2569,571 @@ + return gf; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++llm_graph_params llama_context::graph_params_mtp( ++ llm_graph_result * res, ++ const llama_ubatch & ubatch, ++ const llama_memory_context_i * mctx) const { ++ const llama_model * mtp = model.mtp_assistant.get(); ++ GGML_ASSERT(mtp); ++ ++ return { ++ /*.arch =*/ mtp->arch, ++ /*.hparams =*/ mtp->hparams, ++ /*.cparams =*/ cparams, ++ /*.ubatch =*/ ubatch, ++ /*.gtype =*/ LLM_GRAPH_TYPE_MTP, ++ /*.sched =*/ sched.get(), ++ /*.backend_cpu =*/ backend_cpu, ++ /*.cvec =*/ cvec.get(), ++ /*.loras =*/ loras.get(), ++ /*.mctx =*/ mctx, ++ /*.cross =*/ &cross, ++ /*.samplers =*/ sampling.samplers, ++ /*.n_outputs =*/ n_outputs, ++ /*.cb =*/ graph_get_cb(), ++ /*.res =*/ res, ++ /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) ++ }; ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. ++int32_t llama_context::decode_mtp( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams ++ // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the ++ // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes async→wait. ++ // With no target/draft overlap this is byte-identical to the P3 sync path — the worker runs ++ // the same N-step loop off-thread while we block in decode_mtp_wait. ++ if (out_logits) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_logits, out_h_prev_last); ++ } ++ // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the ++ // fused single-graph draft (decode_mtp_fused) — ONE launch+sync+D2H for the whole block, the ++ // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused ++ // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse → sequential. ++ static const bool fused_enabled = [] { ++ const char * e = getenv("OPENCOTI_MTP_FUSED"); ++ return !(e && e[0] == '0'); ++ }(); ++ if (fused_enabled && n_steps > 1) { ++ return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_h_prev_last); ++ } ++ // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). ++ // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker ++ // gives NO throughput benefit — real depth-2 overlap is deferred (limited by the h-prev ++ // dependency; even the AtomicBot fork ships depth-1) — AND the worker-thread CUDA execution ++ // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off ++ // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. ++ static const bool async_enabled = [] { ++ const char * e = getenv("OPENCOTI_MTP_ASYNC"); ++ return e && e[0] == '1'; ++ }(); ++ if (!async_enabled) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, out_logits, out_h_prev_last); ++ } ++ const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); ++ if (rc != 0) { ++ return rc; ++ } ++ return decode_mtp_wait(out_drafts, out_h_prev_last); ++} ++ ++// opencoti F5 M6-S4 mtp (P2 inert) ++int32_t llama_context::decode_mtp_sync( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, ++ // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) ++ mtp_fused_steps = 1; ++ ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ ++ const int32_t n_vocab = model.vocab.n_tokens(); ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); ++ return -4; ++ } ++ ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(seq_id); ++ data->seq_idx[(size_t) seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post ++ // back). It MUST be flagged output=1 so build_inp_out_ids selects its row — and so the live ++ // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here ++ // produced a divergent graph whose output-selection tensor was left with a null backend ++ // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. ++ data->output[0] = 1; ++ ++ if (n_steps <= 0) { ++ return 0; ++ } ++ ++ // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token ++ // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. ++ for (int32_t k = 0; k < n_steps; ++k) { ++ data->token[0] = last_token; ++ data->pos[0] = attn_pos + 1 + (llama_pos) k; ++ std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); ++ ++ int32_t best_i32 = 0; ++ ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); ++ out_drafts[k] = (llama_token) best_i32; ++ ++ if (out_logits) { ++ ggml_tensor * t_logits = res->get_logits(); ++ GGML_ASSERT(t_logits); ++ ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, ++ 0, (size_t) n_vocab * sizeof(float)); ++ } ++ ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); ++ ++ last_token = (llama_token) best_i32; ++ } ++ ++ if (out_h_prev_last) { ++ std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); ++ } ++ ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head ++// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls ++// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index — already ++// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. ++// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, ++// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). ++// ++// One mask shared across steps (built from step 0's position). For FULL-attn target layers and ++// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa — the short-context / ++// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are ++// byte-identical to the sequential path. For SWA layers at long context the true per-step window ++// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is ++// output-safe — the target verify pass rejects any mismatched draft, so the emitted stream is ++// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix ++// is a per-step mask slice — deferred until measured. See buglog bug-420.) ++int32_t llama_context::decode_mtp_fused( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ if (n_steps <= 0) { ++ return 0; ++ } ++ // A single step has nothing to fuse — defer to the proven sequential path. ++ if (n_steps == 1) { ++ return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, ++ out_drafts, /*out_logits=*/nullptr, out_h_prev_last); ++ } ++ ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); ++ return -4; ++ } ++ ++ // Build/reserve a fused n_steps graph. ++ mtp_fused_steps = n_steps; ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ ++ // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step ++ // query positions consumed by the wrapper's inp_pos_steps (set_input). ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(n_steps); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(seq_id); ++ data->seq_idx[(size_t) seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = (uint32_t) n_steps; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; ++ ++ data->token[0] = last_token; ++ std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ data->pos[k] = attn_pos + 1 + (llama_pos) k; ++ } ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); ++ GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); ++ ++ std::vector drafts((size_t) n_steps); ++ ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ out_drafts[k] = (llama_token) drafts[(size_t) k]; ++ } ++ ++ if (out_h_prev_last) { ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); ++ } ++ ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one ++// in-flight request per context (returns -7 if a prior request was not yet waited). ++int32_t llama_context::decode_mtp_async( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps) { ++ if (!model.mtp_assistant) { ++ LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); ++ return -1; ++ } ++ if (!memory) { ++ LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); ++ return -2; ++ } ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ if (n_bb == 0 || !h_prev || n_steps <= 0) { ++ LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", ++ __func__, n_bb, (const void *) h_prev, n_steps); ++ return -4; ++ } ++ if (!ensure_sched_mtp()) { ++ LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); ++ return -8; ++ } ++ if (!mtp_worker.joinable()) { ++ mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); ++ } ++ ++ { ++ std::unique_lock lk(mtp_mu); ++ if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { ++ LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", ++ __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, ++ (int) mtp_completed.has_value()); ++ return -7; ++ } ++ mtp_request req; ++ req.seq_id = seq_id; ++ req.attn_pos = attn_pos; ++ req.last_token = last_token; ++ req.n_steps = n_steps; ++ req.h_prev.assign(h_prev, h_prev + n_bb); ++ mtp_pending = std::move(req); ++ } ++ mtp_cv_request.notify_one(); ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last ++// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). ++int32_t llama_context::decode_mtp_wait( ++ llama_token * out_drafts, ++ float * out_h_prev_last) { ++ std::unique_lock lk(mtp_mu); ++ mtp_cv_response.wait(lk, [this] { ++ return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); ++ }); ++ if (!mtp_completed.has_value()) { ++ LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); ++ return -7; ++ } ++ mtp_response resp = std::move(*mtp_completed); ++ mtp_completed.reset(); ++ lk.unlock(); ++ ++ if (resp.status != 0) { ++ return resp.status; ++ } ++ if (out_drafts && !resp.drafts.empty()) { ++ std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); ++ } ++ if (out_h_prev_last && !resp.h_prev_last.empty()) { ++ std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); ++ } ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's ++// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the ++// single ubatch row MUST be flagged output — a 0 here reintroduces the null-out-ids abort ++// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. ++int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { ++ auto * kv_iswa = dynamic_cast(memory.get()); ++ if (!kv_iswa) { ++ LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); ++ return -3; ++ } ++ const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; ++ ++ auto data = std::make_shared(); ++ data->token.resize(1); ++ data->embd.resize(n_bb); ++ data->pos.resize(1); ++ data->n_seq_id.resize(1); ++ data->seq_id.resize(1); ++ data->seq_id_data.resize(1); ++ data->output.resize(1); ++ data->seq_idx.resize(LLAMA_MAX_SEQ, -1); ++ data->seq_id_unq.push_back(req.seq_id); ++ data->seq_idx[(size_t) req.seq_id] = 0; ++ ++ llama_ubatch ub{}; ++ ub.b_equal_seqs = 1; ++ ub.n_tokens = 1; ++ ub.n_seq_tokens = 1; ++ ub.n_seqs = 1; ++ ub.n_seqs_unq = 1; ++ ub.n_pos = 1; ++ ub.token = data->token.data(); ++ ub.embd = data->embd.data(); ++ ub.pos = data->pos.data(); ++ ub.n_seq_id = data->n_seq_id.data(); ++ ub.seq_id = data->seq_id.data(); ++ ub.seq_id_unq = data->seq_id_unq.data(); ++ ub.seq_idx = data->seq_idx.data(); ++ ub.output = data->output.data(); ++ ub.data = data; ++ ++ data->n_seq_id[0] = 1; ++ data->seq_id_data[0] = req.seq_id; ++ data->seq_id[0] = &data->seq_id_data[0]; ++ data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 ++ ++ std::vector h(req.h_prev); ++ llama_token last_token = req.last_token; ++ resp.drafts.assign((size_t) req.n_steps, 0); ++ ++ for (int32_t k = 0; k < req.n_steps; ++k) { ++ data->token[0] = last_token; ++ data->pos[0] = req.attn_pos + 1 + (llama_pos) k; ++ std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); ++ ++ llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); ++ if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); ++ return -5; ++ } ++ ++ ggml_status status = GGML_STATUS_SUCCESS; ++ llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); ++ if (!res || status != GGML_STATUS_SUCCESS) { ++ LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); ++ return -6; ++ } ++ ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ ++ ggml_tensor * t_arg = res->get_argmax(); ++ GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); ++ int32_t best_i32 = 0; ++ ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); ++ last_token = (llama_token) best_i32; ++ resp.drafts[(size_t) k] = last_token; ++ ++ ggml_tensor * t_post = res->get_embd(); ++ GGML_ASSERT(t_post); ++ ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); ++ } ++ ++ resp.h_prev_last = std::move(h); ++ return 0; ++} ++ ++// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on ++// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. ++void llama_context::mtp_worker_loop() { ++ for (;;) { ++ mtp_request req; ++ { ++ std::unique_lock lk(mtp_mu); ++ mtp_cv_request.wait(lk, [this] { ++ return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); ++ }); ++ if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { ++ return; ++ } ++ req = std::move(*mtp_pending); ++ mtp_pending.reset(); ++ mtp_in_flight = true; ++ } ++ ++ mtp_response resp; ++ resp.status = decode_mtp_run(req, resp); ++ ++ { ++ std::lock_guard lk(mtp_mu); ++ mtp_in_flight = false; ++ mtp_completed = std::move(resp); ++ } ++ mtp_cv_response.notify_one(); ++ } ++} ++ ++// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV ++// read retired), then sync sched_mtp — WITHOUT consuming mtp_completed (the spec loop's _wait ++// still needs the drafts). Called at decode() top before any KV mutation. Inert when no ++// assistant is loaded or nothing is in flight (drained==false ⇒ no extra sched sync). ++void llama_context::mtp_drain_before_mutate() { ++ if (!model.mtp_assistant) { ++ return; ++ } ++ bool drained = false; ++ { ++ std::unique_lock lk(mtp_mu); ++ if (mtp_pending.has_value() || mtp_in_flight) { ++ mtp_cv_response.wait(lk, [this] { ++ return !mtp_in_flight && !mtp_pending.has_value(); ++ }); ++ drained = true; ++ } ++ } ++ if (drained && sched_mtp) { ++ ggml_backend_sched_synchronize(sched_mtp.get()); ++ } ++} ++ + llm_graph_params llama_context::graph_params( + llm_graph_result * res, + const llama_ubatch & ubatch, +@@ -4043,6 +4830,24 @@ + return ret; + } + ++// opencoti F5 M6-S4 mtp (P2 inert) ++int32_t llama_decode_mtp( ++ llama_context * ctx, ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last) { ++ if (!ctx) { ++ LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); ++ return -1; ++ } ++ return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); ++} ++ + // + // perf + // +diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h +--- a/llama.cpp/src/llama-context.h ++++ b/llama.cpp/src/llama-context.h +@@ -13,6 +13,13 @@ + #include + #include + ++// opencoti F5 M6-S4 mtp (P4): async MTP draft worker thread + single-slot request/response queue. ++#include ++#include ++#include ++#include ++#include ++ + struct llama_model; + class llama_batch_allocr; + +@@ -135,6 +142,65 @@ + llama_memory_context_i * mctx, + ggml_status & ret); + ++ // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. ++ llm_graph_params graph_params_mtp( ++ llm_graph_result * res, ++ const llama_ubatch & ubatch, ++ const llama_memory_context_i * mctx) const; ++ ++ int32_t decode_mtp( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ ++ int32_t decode_mtp_sync( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_logits, ++ float * out_h_prev_last); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps ++ // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft ++ // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced ++ // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits ++ // verification path stays on decode_mtp_sync. ++ int32_t decode_mtp_fused( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ float * h_prev, ++ int32_t n_steps, ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ ++ // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync ++ // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams ++ // no per-step logits); otherwise it submits via decode_mtp_async and blocks in ++ // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the ++ // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). ++ int32_t decode_mtp_async( ++ llama_seq_id seq_id, ++ llama_pos attn_pos, ++ llama_token last_token, ++ const float * h_prev, ++ int32_t n_steps); ++ int32_t decode_mtp_wait( ++ llama_token * out_drafts, ++ float * out_h_prev_last); ++ // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before ++ // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP ++ // assistant is loaded or nothing is in flight (the common case). ++ void mtp_drain_before_mutate(); ++ + int encode(const llama_batch & batch_inp); + int decode(const llama_batch & batch_inp); + +@@ -251,6 +317,14 @@ + const llama_memory_context_i * mctx, + llm_graph_type gtype) const; + ++ // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. ++ bool ensure_sched_mtp(); ++ llm_graph_result * process_ubatch_mtp( ++ const llama_ubatch & ubatch, ++ llama_memory_context_i * mctx, ++ ggml_status & ret); ++ ggml_status graph_compute_mtp(ggml_cgraph * gf); ++ + llm_graph_cb graph_get_cb() const; + + // TODO: read/write lora adapters and cvec +@@ -350,6 +424,46 @@ + llm_graph_result_ptr gf_res_prev; + llm_graph_result_ptr gf_res_reserve; + ++ // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be ++ // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable ++ // single-token MTP graph result across steps of one decode_mtp call. ++ ggml_backend_sched_ptr sched_mtp; ++ llm_graph_result_ptr gf_res_prev_mtp; ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for ++ // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it ++ // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so ++ // ensure_sched_mtp only re-reserves when a larger fused block is requested. ++ int32_t mtp_fused_steps = 1; ++ int32_t mtp_reserved_steps = 0; ++ ++ // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. ++ // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp ++ // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync ++ // facade), so the worker never races the target's sched. The destructor joins the worker. ++ struct mtp_request { ++ llama_seq_id seq_id = 0; ++ llama_pos attn_pos = 0; ++ llama_token last_token = 0; ++ std::vector h_prev; ++ int32_t n_steps = 0; ++ }; ++ struct mtp_response { ++ int32_t status = 0; ++ std::vector drafts; ++ std::vector h_prev_last; ++ }; ++ std::thread mtp_worker; ++ std::atomic mtp_worker_stop{false}; ++ std::mutex mtp_mu; ++ std::condition_variable mtp_cv_request; ++ std::condition_variable mtp_cv_response; ++ std::optional mtp_pending; // submitted, not yet picked up ++ bool mtp_in_flight = false; // worker is processing ++ std::optional mtp_completed; // worker finished, awaiting _wait ++ int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop ++ void mtp_worker_loop(); ++ + // host buffer for the model output (logits and embeddings) + ggml_backend_buffer_ptr buf_output; + +diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp +--- a/llama.cpp/src/llama-graph.cpp ++++ b/llama.cpp/src/llama-graph.cpp +@@ -106,6 +106,38 @@ + return res; + } + ++// opencoti F5 M6-S4 mtp ++void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { ++ GGML_ASSERT(ubatch && ubatch->n_tokens == 1); ++ GGML_ASSERT(ubatch->token && ubatch->embd); ++ GGML_ASSERT(inp_last_token && inp_h_prev); ++ ++ ggml_backend_tensor_set(inp_last_token, ubatch->token, 0, sizeof(llama_token)); ++ ++ const int64_t n_bb = inp_h_prev->ne[0]; ++ ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] ++ // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. ++ for (size_t k = 0; k < inp_pos_steps.size(); ++k) { ++ ggml_backend_tensor_set(inp_pos_steps[k], ubatch->pos + k, 0, sizeof(int32_t)); ++ } ++} ++ ++// opencoti F5 M6-S4 mtp ++bool llm_graph_input_mtp::can_reuse(const llm_graph_params & params) { ++ if (params.gtype != LLM_GRAPH_TYPE_MTP) { ++ return false; ++ } ++ const bool base = inp_last_token && inp_last_token->ne[0] == 1 && inp_h_prev && inp_h_prev->ne[1] == 1; ++ // opencoti F5 M6-S4 mtp (P4 fused): a fused N-step graph and a 1-step graph have different ++ // topology — only reuse when the stored per-step position count matches the requested steps. ++ if (params.n_mtp_steps > 1) { ++ return base && inp_pos_steps.size() == (size_t) params.n_mtp_steps; ++ } ++ return base && inp_pos_steps.empty(); ++} ++ + void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { + // opencoti F5 dca #623: in a DCA-on graph the full-attention layers rope Q from inp_dca's own + // per-regime positions (build_attn_dca), so the standard inp_pos is consumed by no node on archs +@@ -516,35 +548,48 @@ + } + + void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { +- // base tensors may not be allocated if there are no non-SWA attention layers ++ // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently — ++ // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP ++ // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but ++ // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) ++ // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the ++ // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP ++ // case (idxs pruned, mask live) → stale mask, silent divergence (bug-404). Decoupling restores ++ // "set each idxs/mask/rot iff IT is allocated"; for every non-MTP graph idxs+mask share ++ // allocation fate, so this is byte-identical to stock. Matches the AtomicBot fork's guarded iswa. ++ ++ // base index writes (k+v share allocation fate): present iff there are non-SWA write layers. + if (self_k_idxs && self_k_idxs->buffer) { + mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch); + mctx->get_base()->set_input_v_idxs(self_v_idxs, ubatch); +- ++ } ++ // base mask is consumed by the MTP cross-read even when the base idxs are pruned — guard alone. ++ if (self_kq_mask && self_kq_mask->buffer) { + mctx->get_base()->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + } + +- // swa tensors may not be allocated if there are no SWA attention layers ++ // swa index writes (k+v share allocation fate): present iff there are SWA write layers. + if (self_k_idxs_swa && self_k_idxs_swa->buffer) { + mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch); + mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch); +- ++ } ++ if (self_kq_mask_swa && self_kq_mask_swa->buffer) { + mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); + } + +- if (self_k_rot) { ++ if (self_k_rot && self_k_rot->buffer) { + mctx->get_base()->set_input_k_rot(self_k_rot); + } + +- if (self_v_rot) { ++ if (self_v_rot && self_v_rot->buffer) { + mctx->get_base()->set_input_v_rot(self_v_rot); + } + +- if (self_k_rot_swa) { ++ if (self_k_rot_swa && self_k_rot_swa->buffer) { + mctx->get_swa()->set_input_k_rot(self_k_rot_swa); + } + +- if (self_v_rot_swa) { ++ if (self_v_rot_swa && self_v_rot_swa->buffer) { + mctx->get_swa()->set_input_v_rot(self_v_rot_swa); + } + } +@@ -844,6 +889,7 @@ + t_logits = nullptr; + t_embd = nullptr; + t_embd_pooled = nullptr; ++ t_argmax = nullptr; // opencoti F5 M6-S4 mtp + t_sampled.clear(); + t_sampled_probs.clear(); + t_sampled_logits.clear(); +@@ -2747,6 +2793,91 @@ + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } ++ ++ return cur; ++} ++ ++// opencoti F5 M6-S4 mtp ++ggml_tensor * llm_graph_context::build_attn_mtp( ++ llm_graph_input_attn_kv_iswa * inp, ++ ggml_tensor * wo, ++ ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ++ ggml_tensor * kq_b, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il_mtp, ++ int32_t il_kv_tgt, ++ bool read_from_swa_kv, ++ int64_t kv_embd_head_v, ++ int64_t kv_n_head_v, ++ bool use_k_as_v) const { ++ auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; ++ auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; ++ ++ if (k_rot) { ++ q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); ++ } ++ if (v_rot) { ++ // V-only rotation is applied after MHA in the standard path; no V write here. ++ } ++ ++ ggml_build_forward_expand(gf, q_cur); ++ ++ const auto * mctx_iswa = inp->mctx; ++ const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); ++ ++ const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); ++ ++ ggml_tensor * q = q_cur; ++ ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); ++ ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); ++ ++ if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) { ++ // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the ++ // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp ++ // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at ++ // head_dim∈{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 ++ // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired ++ // inverse on the output → wrong logits (graph still *built*, which is why P2's inert build ++ // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity ++ // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. ++ if (q->ne[0] % 128 != 0) { ++ const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; ++ q = ggml_pad(ctx0, q, pad, 0, 0, 0); ++ } ++ if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } ++ } ++ ++ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); ++ cb(cur, "kqv_out_mtp", il_mtp); ++ ++ if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0) { ++ const int64_t orig_v_head = kv_embd_head_v; ++ const int64_t padded_v_head = v->ne[0]; ++ if (padded_v_head != orig_v_head) { ++ const int64_t n_head_v = kv_n_head_v; ++ const int64_t n_tokens_cur = cur->ne[1]; ++ cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); ++ cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, ++ cur->nb[1], cur->nb[2], 0); ++ cur = ggml_cont(ctx0, cur); ++ cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); ++ } ++ } ++ ++ if (v_rot) { ++ cur = ggml_mul_mat_aux(ctx0, cur, v_rot); ++ } ++ ++ if (wo) { ++ cur = build_lora_mm(wo, cur); ++ cb(cur, "mtp_wo_out", il_mtp); ++ } ++ if (wo_b) { ++ cur = ggml_add(ctx0, cur, wo_b); ++ } + + return cur; + } +diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h +--- a/llama.cpp/src/llama-graph.h ++++ b/llama.cpp/src/llama-graph.h +@@ -36,6 +36,7 @@ + LLM_GRAPH_TYPE_ENCODER, + LLM_GRAPH_TYPE_DECODER, + LLM_GRAPH_TYPE_DECODER_MTP, ++ LLM_GRAPH_TYPE_MTP, // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter graph (distinct from native DECODER_MTP/NextN) + }; + + enum llm_ffn_op_type { +@@ -124,6 +125,25 @@ + const int64_t n_embd = 0; + }; + ++// opencoti F5 M6-S4 mtp ++// Gemma 4 MTP: last target token id + backbone hidden (n_bb floats) for a single step. ++class llm_graph_input_mtp : public llm_graph_input_i { ++public: ++ llm_graph_input_mtp() = default; ++ ~llm_graph_input_mtp() override = default; ++ ++ void set_input(const llama_ubatch * ubatch) override; ++ bool can_reuse(const llm_graph_params & params) override; ++ ++ ggml_tensor * inp_last_token = nullptr; // I32 [1] ++ ggml_tensor * inp_h_prev = nullptr; // F32 [n_bb, 1] ++ // opencoti F5 M6-S4 mtp (P4 fused): one I32[1] RoPE position per fused draft step. ++ // Empty for the single-step path (which uses build_inp_pos). For n_mtp_steps>1 the draft ++ // chain is unrolled in-graph: each step's token/h chain from the previous step's argmax/ ++ // backbone, but the per-step query position still comes from the host via these tensors. ++ std::vector inp_pos_steps; ++}; ++ + class llm_graph_input_pos : public llm_graph_input_i { + public: + llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} +@@ -572,9 +592,20 @@ + + llm_graph_result * res; + ++ // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. ++ // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone ++ // into the next step with no host round-trip / per-step GPU sync (the throughput fix). ++ // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. ++ int32_t n_mtp_steps = 1; ++ + // return true if the "other" params would result in a graph with the same topology as with the current params + // having the same topology allows us to reuse the graph in some cases + bool allow_reuse(const llm_graph_params & other) const { ++ // opencoti F5 M6-S4 mtp: 0.10.3's native allow_reuse condition below already permits reuse when ++ // BOTH token+embd are present in both ubatches (the third OR-clause) — exactly the Gemma-4 MTP ++ // case — so the 0.10.1-era token_compat relaxation is redundant and intentionally not restored. ++ // The fused n_mtp_steps>1 graph reuse (the P4 throughput win) is preserved by that native clause. ++ + // first check the ubatch + bool can_reuse_ubatch = + ubatch.equal_seqs() == other.ubatch.equal_seqs() && +@@ -651,6 +682,12 @@ + ggml_tensor * get_embd_pooled() const { return t_embd_pooled; } + ggml_tensor * get_h_pre_norm() const { return t_h_pre_norm; } + ++ // opencoti F5 M6-S4 mtp ++ // Optional in-graph argmax tensor (I32 [n_outputs]). Currently set only by the ++ // Gemma4 MTP graph so the host can read a 4-byte argmax instead of an ++ // n_vocab-float logits row + CPU argmax loop. ++ ggml_tensor * get_argmax() const { return t_argmax; } ++ + ggml_cgraph * get_gf() const { return gf; } + ggml_context * get_ctx() const { return ctx_compute.get(); } + +@@ -679,6 +716,7 @@ + ggml_tensor * t_embd = nullptr; + ggml_tensor * t_embd_pooled = nullptr; + ggml_tensor * t_h_pre_norm = nullptr; // [n_embd, n_outputs] hidden state before final output norm ++ ggml_tensor * t_argmax = nullptr; // opencoti F5 M6-S4 mtp: optional, currently MTP-only (see get_argmax) + + std::map t_sampled_logits; + std::map t_candidates; +@@ -983,6 +1021,26 @@ + int il, + bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) + ++ // opencoti F5 M6-S4 mtp ++ // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). ++ // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). ++ // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). ++ ggml_tensor * build_attn_mtp( ++ llm_graph_input_attn_kv_iswa * inp, ++ ggml_tensor * wo, ++ ggml_tensor * wo_b, ++ ggml_tensor * q_cur, ++ ggml_tensor * kq_b, ++ ggml_tensor * sinks, ++ ggml_tensor * v_mla, ++ float kq_scale, ++ int il_mtp, ++ int32_t il_kv_tgt, ++ bool read_from_swa_kv, ++ int64_t kv_embd_head_v, ++ int64_t kv_n_head_v, ++ bool use_k_as_v) const; ++ + llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; + + ggml_tensor * build_attn( +diff --git a/llama.cpp/src/llama-hparams.h b/llama.cpp/src/llama-hparams.h +--- a/llama.cpp/src/llama-hparams.h ++++ b/llama.cpp/src/llama-hparams.h +@@ -217,6 +217,14 @@ + // gemma4 per-layer embedding + uint32_t n_embd_per_layer = 0; + ++ // opencoti F5 M6-S4 mtp — gemma4 assistant-MTP (speculative drafter), opencoti-private fields. ++ // Backbone width is NOT a private field: it reuses n_embd_out_impl (above), loaded from the generic ++ // upstream KV %s.embedding_length_out (#23398) — so this struct stays free of a redundant duplicate. ++ uint32_t n_centroids = 0; ++ uint32_t centroid_top_k = 0; ++ bool attention_k_eq_v = false; ++ bool use_ordered_embeddings = false; ++ + // needed by encoder-decoder models (e.g. T5, FLAN-T5) + // ref: https://github.com/ggml-org/llama.cpp/pull/8141 + llama_token dec_start_token_id = LLAMA_TOKEN_NULL; +diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp +--- a/llama.cpp/src/llama-kv-cache-iswa.cpp ++++ b/llama.cpp/src/llama-kv-cache-iswa.cpp +@@ -229,6 +229,21 @@ + return std::make_unique(this); + } + ++// opencoti F5 M6-S4 mtp ++llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { ++ llama_kv_cache::slot_info_vec_t sinfos_base; ++ llama_kv_cache::slot_info_vec_t sinfos_swa; ++ ++ sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); ++ sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); ++ ++ std::vector ubatches; ++ ubatches.push_back(std::move(ubatch)); ++ ++ return std::make_unique( ++ this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); ++} ++ + llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); + } +diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h +--- a/llama.cpp/src/llama-kv-cache-iswa.h ++++ b/llama.cpp/src/llama-kv-cache-iswa.h +@@ -60,6 +60,9 @@ + + llama_memory_context_ptr init_full() override; + ++ // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). ++ llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); ++ + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; +diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp +--- a/llama.cpp/src/llama-kv-cache.cpp ++++ b/llama.cpp/src/llama-kv-cache.cpp +@@ -1827,6 +1827,39 @@ + return updated; + } + ++// opencoti F5 M6-S4 mtp ++llama_kv_cache::slot_info llama_kv_cache::mtp_slot_info(llama_seq_id seq_id) const { ++ GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); ++ ++ const uint32_t st = seq_to_stream[seq_id]; ++ const auto & cells = v_cells[st]; ++ ++ llama_pos pmax = cells.seq_pos_max(seq_id); ++ ++ uint32_t idx = 0; ++ ++ if (pmax >= 0) { ++ for (uint32_t i = 0; i < cells.size(); ++i) { ++ if (!cells.seq_has(i, seq_id)) { ++ continue; ++ } ++ if (cells.pos_get(i) == pmax) { ++ idx = i; ++ break; ++ } ++ } ++ } ++ ++ slot_info res; ++ res.s0 = 0; ++ res.s1 = 0; ++ res.strm = { (llama_seq_id) st }; ++ res.idxs.resize(1); ++ res.idxs[0] = { idx }; ++ ++ return res; ++} ++ + llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, bool cont) const { + + if (debug > 0) { +@@ -5039,6 +5072,16 @@ + llama_kv_cache * kv, + llama_kv_cache::slot_info_vec_t sinfos, + std::vector ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { ++ // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, ++ // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from ++ // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a ++ // view with a garbage dim-2 (observed ne[2]=2956782432 → ggml.c view-bounds assert). For ++ // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. ++ if (!this->sinfos.empty()) { ++ n_kv = (int32_t) kv->get_n_kv(this->sinfos[0]); ++ } else { ++ n_kv = 0; ++ } + } + + llama_kv_cache_context::~llama_kv_cache_context() = default; +diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h +--- a/llama.cpp/src/llama-kv-cache.h ++++ b/llama.cpp/src/llama-kv-cache.h +@@ -288,6 +288,10 @@ + // return empty vector on failure + slot_info_vec_t prepare(const std::vector & ubatches); + ++ // opencoti F5 M6-S4 mtp: read-only slot_info addressing the last resident cell of seq_id. ++ // Used by the MTP cross-read path (no KV write); points the MTP graph at the target cache. ++ slot_info mtp_slot_info(llama_seq_id seq_id) const; ++ + bool update(llama_context * lctx, bool do_shift, const stream_copy_info & sc_info); + + // find a slot of kv cells that can hold the ubatch +diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp +--- a/llama.cpp/src/llama-model.cpp ++++ b/llama.cpp/src/llama-model.cpp +@@ -136,6 +136,8 @@ + return new llama_model_gemma3n(params); + case LLM_ARCH_GEMMA4: + return new llama_model_gemma4(params); ++ case LLM_ARCH_GEMMA4_ASSISTANT: // opencoti F5 M6-S4 mtp: Gemma-4 MTP assistant (loaded into target) ++ return new llama_model_gemma4_assistant(params); + case LLM_ARCH_GEMMA_EMBEDDING: + return new llama_model_gemma_embedding(params); + case LLM_ARCH_STARCODER2: +@@ -1577,6 +1579,19 @@ + tn, ne, flags); + } + ++// opencoti F5 M6-S4 mtp (#408/bug-408): force OUTPUT-buft placement for a tensor that classifies as ++// LLM_TENSOR_LAYER_INPUT. ml.create_tensor() selects the buft_list by the tensor's llm_tensor_info layer; ++// TOKEN_EMBD is LAYER_INPUT, so passing dev_output.buft_list in the INPUT slot routes it onto the GPU ++// output buft. Used for the gemma4-assistant TIED OUTPUT head so the per-step full-vocab build_lora_mm ++// matmul stays on GPU (the P4/#408 win the 0.10.3 bump dropped — see src/models/gemma4-assistant.cpp). ++// NOT TENSOR_DUPLICATED: the assistant's token_embd is the SOLE use of that GGUF tensor (output head ++// only; input embeddings come from the TARGET), so it must use normal create/load accounting. ++ggml_tensor * llama_model_base::create_tensor_output_head(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { ++ return ml.create_tensor( ++ hparams, &pimpl->cpu_buft_list, pimpl->dev_output.buft_list, pimpl->dev_output.buft_list, nullptr, ++ tn, ne, flags); ++} ++ + std::string llama_model::arch_name() const { + return llm_arch_name(arch); + } +@@ -2349,6 +2364,7 @@ + case LLM_ARCH_GEMMA3: + case LLM_ARCH_GEMMA3N: + case LLM_ARCH_GEMMA4: ++ case LLM_ARCH_GEMMA4_ASSISTANT: // opencoti F5 M6-S4 mtp + case LLM_ARCH_GEMMA_EMBEDDING: + case LLM_ARCH_STARCODER2: + case LLM_ARCH_OPENELM: +@@ -2564,6 +2580,12 @@ + return create_tensor(*ml, tn, ne, flags); + } + ++// opencoti F5 M6-S4 mtp (#408/bug-408): ml-less convenience overload, mirroring create_tensor above. ++ggml_tensor * llama_model_base::create_tensor_output_head(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { ++ GGML_ASSERT(ml != nullptr); ++ return create_tensor_output_head(*ml, tn, ne, flags); ++} ++ + void llama_model_base::create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) { + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED); + if (layer.ffn_gate_up_exps == nullptr) { +diff --git a/llama.cpp/src/llama-model.h b/llama.cpp/src/llama-model.h +--- a/llama.cpp/src/llama-model.h ++++ b/llama.cpp/src/llama-model.h +@@ -559,6 +559,17 @@ + struct ggml_tensor * per_layer_model_proj = nullptr; + struct ggml_tensor * per_layer_proj_norm = nullptr; + ++ // opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant weights ++ // (populated only when arch == GEMMA4_ASSISTANT nested model) ++ struct ggml_tensor * mtp_pre_projection = nullptr; ++ struct ggml_tensor * mtp_post_projection = nullptr; ++ struct ggml_tensor * mtp_centroids = nullptr; ++ struct ggml_tensor * mtp_token_ordering = nullptr; ++ ++ // opencoti F5 M6-S4 mtp — nested assistant model loaded via ++ // llama_model_load_mtp_from_file (owns the mtp_* tensors above) ++ std::unique_ptr mtp_assistant; ++ + std::vector layers; + + //Dense linear projections for SentenceTransformers models like embeddinggemma +@@ -669,6 +680,14 @@ + // convenience overload of create_tensor that doesn't require llama_model_loader + ggml_tensor * create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); + ++ // opencoti F5 M6-S4 mtp (#408/bug-408): create a tensor on the OUTPUT device buft regardless of its ++ // classification. For the gemma4-assistant TIED OUTPUT head (tok_embd, used ONLY as the per-step ++ // full-vocab build_lora_mm weight — never an input get_rows) this keeps the matmul on the GPU instead ++ // of a graph-fragmenting CPU island. Forwards dev_output.buft_list in the input slot (TOKEN_EMBD ++ // classifies LAYER_INPUT) with NORMAL load accounting (no TENSOR_DUPLICATED — sole use of the tensor). ++ ggml_tensor * create_tensor_output_head(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); ++ ggml_tensor * create_tensor_output_head(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); ++ + // helper: try merged gate_up_exps first, fall back to separate gate and up + void create_tensor_gate_up_exps(llama_layer & layer, int bid, int64_t n_embd_, + int64_t n_ff_, int64_t n_expert_, int flags); +diff --git a/llama.cpp/src/llama.cpp b/llama.cpp/src/llama.cpp +--- a/llama.cpp/src/llama.cpp ++++ b/llama.cpp/src/llama.cpp +@@ -453,6 +453,120 @@ + return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params); + } + ++// opencoti F5 M6-S4 mtp — Gemma 4 MTP assistant load surface ++static bool llama_mtp_vocab_matches(const llama_model & tgt, const llama_model & aux) { ++ const llama_vocab & vt = tgt.vocab; ++ const llama_vocab & va = aux.vocab; ++ if (vt.n_tokens() != va.n_tokens()) { ++ LLAMA_LOG_ERROR("%s: vocab size mismatch (target=%u assistant=%u)\n", __func__, vt.n_tokens(), va.n_tokens()); ++ return false; ++ } ++ // opencoti F5 M6-S4 mtp: the gemma4_assistant draft head ships NO tokenizer of its own ++ // (vocab type NONE) — it shares the target's vocab, and its token_ordering/centroids are sized ++ // to the shared vocab. token_get_text() asserts on a NONE vocab (no id_to_token table), so the ++ // per-token text comparison below is both impossible and meaningless for the vocab-less draft. ++ // The n_tokens() equality above is the meaningful invariant; accept once it holds. ++ if (va.get_type() == LLAMA_VOCAB_TYPE_NONE) { ++ return true; ++ } ++ constexpr int32_t k_check_from = 5; // align with speculative MTP vocab check ++ for (uint32_t i = (uint32_t) k_check_from; i < vt.n_tokens(); ++i) { ++ const llama_token id = (llama_token) i; ++ if (std::strcmp(vt.token_get_text(id), va.token_get_text(id)) != 0) { ++ LLAMA_LOG_ERROR("%s: vocab text mismatch at token id %d\n", __func__, (int) id); ++ return false; ++ } ++ } ++ return true; ++} ++ ++int llama_model_load_mtp_from_file(struct llama_model * model, const char * path_mtp, struct llama_model_params params) { ++ if (!model || !path_mtp || !path_mtp[0]) { ++ LLAMA_LOG_ERROR("%s: invalid arguments\n", __func__); ++ return -1; ++ } ++ ++ llama_model * tgt = (llama_model *) model; ++ if (tgt->arch != LLM_ARCH_GEMMA4) { ++ LLAMA_LOG_ERROR("%s: MTP target must be arch gemma4 (got %s)\n", __func__, llm_arch_name(tgt->arch)); ++ return -2; ++ } ++ ++ llama_model * aux = llama_model_load_from_file(path_mtp, params); ++ if (!aux) { ++ LLAMA_LOG_ERROR("%s: failed to load assistant from '%s'\n", __func__, path_mtp); ++ return -3; ++ } ++ ++ if (aux->arch != LLM_ARCH_GEMMA4_ASSISTANT) { ++ LLAMA_LOG_ERROR("%s: MTP weights must be arch gemma4-assistant (got %s)\n", __func__, llm_arch_name(aux->arch)); ++ llama_model_free(aux); ++ return -4; ++ } ++ ++ // opencoti F5 M6-S4 mtp: arch prefix now hyphenated (upstream-aligned "gemma4-assistant"); the ++ // GGUF key prefix follows the arch string, so this raw lookup must match. ++ const auto req_it = aux->gguf_kv.find("gemma4-assistant.requires_target_arch"); ++ if (req_it != aux->gguf_kv.end() && req_it->second != "gemma4") { ++ LLAMA_LOG_ERROR("%s: assistant requires_target_arch='%s' (expected gemma4)\n", __func__, req_it->second.c_str()); ++ llama_model_free(aux); ++ return -5; ++ } ++ ++ if (aux->hparams.n_embd_out_impl == 0 || aux->hparams.n_embd_out_impl != tgt->hparams.n_embd) { ++ LLAMA_LOG_ERROR("%s: assistant backbone width (embedding_length_out) %u must match target n_embd %u\n", __func__, ++ aux->hparams.n_embd_out_impl, tgt->hparams.n_embd); ++ llama_model_free(aux); ++ return -6; ++ } ++ ++ if (!llama_mtp_vocab_matches(*tgt, *aux)) { ++ llama_model_free(aux); ++ return -7; ++ } ++ // opencoti F5 M6-S4 mtp: namespace ALL assistant tensors under an in-memory "mtp." prefix so -ot ++ // rules can target them without colliding with the target model's identically-named generic tensors ++ // (the assistant has its own blk.*, token_embd, output_norm). This is a post-load in-memory rename ++ // for -ot ONLY; it does NOT touch GGUF identity — on-disk names stay upstream-aligned (nextn.* / ++ // masked_embd_*, none of which start with "mtp."), so every entry is prefixed exactly once and the ++ // existing "mtp.*" wildcard -ot rules keep matching. ++ for (auto & kv : aux->tensors_by_name) { ++ if (kv.first.substr(0, 4) != "mtp.") { ++ std::string new_name = "mtp." + kv.first; ++ ggml_set_name(kv.second, new_name.c_str()); ++ kv.first = new_name; ++ } ++ } ++ tgt->mtp_assistant.reset(aux); ++ return 0; ++} ++ ++const struct llama_model * llama_model_get_mtp_assistant(const struct llama_model * model) { ++ if (!model) { ++ return nullptr; ++ } ++ const llama_model * m = (const llama_model *) model; ++ return m->mtp_assistant.get(); ++} ++ ++bool llama_model_has_mtp_assistant(const struct llama_model * model) { ++ return llama_model_get_mtp_assistant(model) != nullptr; ++} ++ ++// opencoti F5 M6-S4 mtp: fork-private C API (no upstream counterpart — upstream uses ctx_other and ++// never exposes the assistant backbone width). Name kept for ABI stability; the backing hparams field ++// is now n_embd_out_impl (upstream-aligned, from %s.embedding_length_out). ++uint32_t llama_model_mtp_n_embd_backbone(const struct llama_model * model) { ++ if (!model) { ++ return 0; ++ } ++ const llama_model * m = (const llama_model *) model; ++ if (!m->mtp_assistant) { ++ return 0; ++ } ++ return m->mtp_assistant->hparams.n_embd_out_impl; ++} ++ + void llama_model_save_to_file(const struct llama_model * model, const char * path_model) { + llama_model_saver ms(model); + ms.add_kv_from_model(); +diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp +--- a/llama.cpp/src/models/gemma4.cpp ++++ b/llama.cpp/src/models/gemma4.cpp +@@ -132,6 +132,14 @@ + } + + std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { ++ // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), ++ // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF ++ // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ ++ // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. ++ if (params.gtype == LLM_GRAPH_TYPE_MTP) { ++ GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); ++ return std::make_unique(*this, *mtp_assistant, params); ++ } + return std::make_unique(*this, params); + } + +diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h +--- a/llama.cpp/src/models/models.h ++++ b/llama.cpp/src/models/models.h +@@ -809,6 +809,26 @@ + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; + }; + ++// opencoti F5 M6-S4 mtp ++// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. ++struct llm_build_gemma4_mtp : public llm_graph_context { ++ const llama_model & target; ++ const llama_model & mtp; ++ ++ llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); ++}; ++ ++// opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant — loaded INTO a gemma4 target via ++// llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own ++// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the ++// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). ++struct llama_model_gemma4_assistant : public llama_model_base { ++ llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} ++ void load_arch_hparams(llama_model_loader & ml) override; ++ void load_arch_tensors(llama_model_loader & ml) override; ++ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; ++}; ++ + + struct llama_model_gemma_embedding : public llama_model_base { + llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {} +diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp +--- a/llama.cpp/tools/server/server-context.cpp ++++ b/llama.cpp/tools/server/server-context.cpp +@@ -806,7 +806,19 @@ + const bool spec_mtp = std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end(); +- const bool has_draft = params_base.speculative.has_dft(); ++ // opencoti F5 M6-S4 mtp (#485): the Gemma assistant draft (COMMON_SPECULATIVE_TYPE_MTP) is a ++ // separate GGUF (so has_dft()==true) but CANNOT be loaded standalone — gemma4-assistant's ++ // build_arch_graph throws "cannot be used as a primary model". The old code fell into the ++ // has_draft measure branch below, called common_get_device_memory_data() on it, threw, and ++ // logged a misleading "[spec] failed to measure draft model memory" (it reads as a boot error ++ // and tripped probe readiness loops -> false BOOT-FAIL, the bug-550 misdiagnosis). The ++ // assistant head loads INTO the target model post-fit and is small, so skip the standalone ++ // reserve entirely for the assistant engine (the caught-exception path already added 0 bytes; ++ // this just makes it explicit and silences the bogus error). ++ const bool spec_assistant_fit = std::find(params_base.speculative.types.begin(), ++ params_base.speculative.types.end(), ++ COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); ++ const bool has_draft = params_base.speculative.has_dft() && !spec_assistant_fit; + + if (has_draft || spec_mtp) { + common_params params_dft = params_base; +@@ -906,7 +918,58 @@ + + add_bos_token = llama_vocab_get_add_bos(vocab); + +- if (params_base.speculative.has_dft()) { ++ // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so ++ // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file — ++ // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at ++ // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from ++ // both native branches (separate-model draft below; NextN MTP-ctx-against-target further ++ // down). Detect it via the enabled types vector, load into the target, and leave ++ // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's ++ // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). ++ const bool spec_assistant = std::find(params_base.speculative.types.begin(), ++ params_base.speculative.types.end(), ++ COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); ++ if (params_base.speculative.has_dft() && spec_assistant) { ++ // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target ++ // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the ++ // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout ++ // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. ++ // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 ++ // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it ++ // explicitly and fail clean at boot instead of crashing mid-request. ++ if (params_base.n_parallel > 1 && !params_base.kv_unified) { ++ SRV_ERR("%s", ++ "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " ++ "--kv-unified: the per-stream-split KV layout is not supported by the single-context " ++ "MTP cross-read. Re-run with --kv-unified (PolyKV already sets it).\n"); ++ return false; ++ } ++ ++ const auto & params_spec = params_base.speculative.draft; ++ ++ SRV_INF("loading MTP assistant '%s' into target\n", params_spec.mparams.path.c_str()); ++ ++ auto params_dft = params_base; ++ params_dft.devices = params_spec.devices; ++ params_dft.model = params_spec.mparams; ++ params_dft.n_gpu_layers = params_spec.n_gpu_layers; ++ params_dft.cache_type_k = params_spec.cache_type_k; ++ params_dft.cache_type_v = params_spec.cache_type_v; ++ params_dft.tensor_buft_overrides = params_spec.tensor_buft_overrides; ++ ++ auto mparams_dft = common_model_params_to_llama(params_dft); ++ ++ const int mtp_rc = llama_model_load_mtp_from_file(model_tgt, params_dft.model.path.c_str(), mparams_dft); ++ if (mtp_rc != 0) { ++ SRV_ERR("failed to load MTP assistant '%s' (rc=%d)\n", params_dft.model.path.c_str(), mtp_rc); ++ return false; ++ } ++ ++ // No separate draft model/context: the assistant decodes on ctx_tgt via llama_decode_mtp. ++ params_base.speculative.draft.ctx_tgt = ctx_tgt; ++ params_base.speculative.draft.ctx_dft = nullptr; ++ SRV_INF("%s", "MTP assistant loaded into target\n"); ++ } else if (params_base.speculative.has_dft()) { + // TODO speculative: move to common/speculative.cpp? + const auto & params_spec = params_base.speculative.draft; + +diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp +new file mode 100644 +--- /dev/null ++++ b/llama.cpp/src/models/gemma4-assistant.cpp +@@ -0,0 +1,488 @@ ++// opencoti F5 M6-S4 mtp ++#include "models.h" ++ ++#include ++#include ++ ++static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { ++ p.arch = mtp_model.arch; ++ p.hparams = mtp_model.hparams; ++ p.gtype = LLM_GRAPH_TYPE_MTP; ++ return p; ++} ++ ++// Last layer in [range_start, range_end) whose attention type matches want_swa. ++static int32_t gemma4_mtp_kv_layer_last_in_range( ++ const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { ++ int32_t best = -1; ++ if (range_start < 0) { ++ range_start = 0; ++ } ++ if (range_end > (int32_t) tgt.n_layer) { ++ range_end = (int32_t) tgt.n_layer; ++ } ++ for (int32_t il = range_start; il < range_end; ++il) { ++ if (tgt.is_swa((uint32_t) il) == want_swa) { ++ best = il; ++ } ++ } ++ return best; ++} ++ ++// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> ++// (post-projected backbone hidden, vocabulary logits, argmax token id). ++// ++// Used by both the single-step path (n_mtp_steps==1) and the chained path ++// (n_mtp_steps>1, called n_mtp_steps times within the same graph). ++// ++// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position ++// for this step's RoPE (the cross-attn mask is shared across steps because the ++// target's KV cache only contains positions ≤ attn_pos and all step positions ++// are > attn_pos, so causal/SWA admit all target cells uniformly). ++// ++// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too — required for greedy match with verify). ++// `out_argmax` (I32 [1]): greedy token id on-device. ++static void gemma4_mtp_build_one_step( ++ const llm_graph_context & gctx, ++ const llama_model & target, ++ const llama_model & mtp, ++ llm_graph_input_attn_kv_iswa * inp_attn, ++ ggml_tensor * tok_step, // I32 [1] ++ ggml_tensor * h_step, // F32 [n_bb, 1] ++ ggml_tensor * pos_step, // I32 [1] ++ ggml_tensor ** out_logits, // F32 [n_vocab, 1] ++ ggml_tensor ** out_h_post, // F32 [n_bb, 1] ++ ggml_tensor ** out_argmax) { // I32 [1] ++ auto * ctx0 = gctx.ctx0; ++ auto * gf = gctx.gf; ++ const auto & hparams = gctx.hparams; ++ const auto & cparams = gctx.cparams; ++ const int n_layer = (int) gctx.n_layer; ++ const int n_tokens = (int) gctx.n_tokens; ++ const int n_ctx_orig = (int) gctx.n_ctx_orig; ++ const int rope_type = gctx.rope_type; ++ const float ext_factor = gctx.ext_factor; ++ const float attn_factor = gctx.attn_factor; ++ const float beta_fast = gctx.beta_fast; ++ const float beta_slow = gctx.beta_slow; ++ auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; ++ ++ const float tok_scale = sqrtf((float) target.hparams.n_embd); ++ ++ ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); ++ cb(tok_e, "mtp_tgt_tok_embd", -1); ++ ++ // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). ++ // Use target n_embd so Edge / non-Edge targets match the main forward. ++ tok_e = ggml_scale(ctx0, tok_e, tok_scale); ++ cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); ++ ++ ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); ++ cb(inp_cat, "mtp_concat", -1); ++ ++ ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); ++ cb(inpL, "mtp_pre_proj_out", -1); ++ ++ ggml_build_forward_expand(gf, inpL); ++ ++ ggml_tensor * cur = nullptr; ++ ++ for (int il = 0; il < n_layer; ++il) { ++ const int64_t n_embd_head = hparams.n_embd_head_k(il); ++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); ++ ++ const int64_t n_head = hparams.n_head(il); ++ ++ const float freq_base_l = mtp.get_rope_freq_base(cparams, il); ++ const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); ++ const int n_rot_l = hparams.n_rot(il); ++ ++ cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "attn_norm", il); ++ ++ ggml_tensor * freq_factors = nullptr; ++ if (!hparams.is_swa(il)) { ++ freq_factors = mtp.layers[il].rope_freqs; ++ } ++ ++ ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); ++ cb(Qcur, "Qcur", il); ++ ++ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); ++ ++ Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); ++ cb(Qcur, "Qcur_normed", il); ++ ++ Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ++ ext_factor, attn_factor, beta_fast, beta_slow); ++ cb(Qcur, "Qcur_pos", il); ++ ++ const bool read_swa = hparams.is_swa(il); ++ ++ const int32_t n_tgt = (int32_t) target.hparams.n_layer; ++ ++ // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads ++ // ONE shared KV per attention type from the target — the LAST layer of that type. ++ // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py ++ // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} ++ const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); ++ ++ GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); ++ ++ // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and ++ // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the ++ // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must ++ // ALWAYS fetch V from the cache — not reuse the post-RoPE K tensor. ++ const bool use_k_as_v = false; ++ ++ const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); ++ const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); ++ ++ cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, ++ hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); ++ ++ cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "attn_post_norm", il); ++ ++ ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); ++ cb(attn_out, "attn_out", il); ++ ++ GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); ++ ++ cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); ++ cb(cur, "ffn_norm", il); ++ ++ cur = gctx.build_ffn(cur, ++ mtp.layers[il].ffn_up, nullptr, nullptr, ++ mtp.layers[il].ffn_gate, nullptr, nullptr, ++ mtp.layers[il].ffn_down, nullptr, nullptr, ++ nullptr, ++ LLM_FFN_GELU, LLM_FFN_PAR, il); ++ cb(cur, "ffn_out", il); ++ ++ cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "ffn_post_norm", il); ++ ++ cur = ggml_add(ctx0, cur, attn_out); ++ ++ if (mtp.layers[il].out_scale) { ++ cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); ++ cb(cur, "out_scaled", il); ++ } ++ ++ cur = gctx.build_cvec(cur, il); ++ cb(cur, "l_out", il); ++ ++ inpL = cur; ++ } ++ ++ cur = inpL; ++ ++ cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); ++ cb(cur, "result_norm", -1); ++ ++ ggml_tensor * h_inner = cur; ++ ++ ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); ++ cb(backbone, "mtp_post_proj_out", -1); ++ ++ const int64_t n_vocab = mtp.tok_embd->ne[1]; ++ const int64_t n_tokens_mtp = h_inner->ne[1]; ++ ++ if (mtp.hparams.use_ordered_embeddings) { ++ // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full ++ // [n_vocab] row then argmax — matches masked full-vocab greedy (sparse-only argmax broke server accept). ++ GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); ++ GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); ++ const uint32_t n_c = mtp.hparams.n_centroids; ++ const uint32_t top_k = mtp.hparams.centroid_top_k; ++ GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); ++ GGML_ASSERT(n_vocab % (int64_t) n_c == 0); ++ const int64_t vsc = n_vocab / (int64_t) n_c; ++ ++ ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); ++ cb(centroid_logits, "mtp_centroid_logits", -1); ++ ++ ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); ++ cb(topk_idx, "mtp_centroid_topk_idx", -1); ++ ++ const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); ++ ggml_tensor * ordering = ggml_view_2d( ++ ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); ++ cb(ordering, "mtp_token_ordering_view", -1); ++ ++ ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); ++ cb(sel_ids, "mtp_selected_token_ids", -1); ++ ++ const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; ++ ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); ++ cb(flat_ids, "mtp_selected_token_ids_flat", -1); ++ ++ ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); ++ cb(sel_emb, "mtp_selected_embd", -1); ++ ++ ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); ++ cb(sel_logits, "mtp_selected_logits", -1); ++ ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); ++ ++ ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); ++ logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); ++ cb(logits_full, "mtp_logits_masked_base", -1); ++ ++ ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); ++ ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); ++ cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); ++ cb(cur, "mtp_logits_scatter_view", -1); ++ cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); ++ cb(cur, "mtp_logits_full", -1); ++ } else { ++ cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); ++ cb(cur, "result_output_dense", -1); ++ } ++ ++ if (hparams.f_final_logit_softcapping) { ++ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); ++ cur = ggml_tanh(ctx0, cur); ++ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); ++ } ++ ++ cb(cur, "result_output", -1); ++ ++ // Greedy argmax on-device: I32 [1] token index into the vocabulary row. ++ ggml_tensor * arg = ggml_argmax(ctx0, cur); ++ cb(arg, "result_argmax", -1); ++ ++ *out_logits = cur; ++ *out_h_post = backbone; ++ *out_argmax = arg; ++} ++ ++llm_build_gemma4_mtp::llm_build_gemma4_mtp( ++ const llama_model & target_model, ++ const llama_model & mtp_model, ++ const llm_graph_params & params) : ++ llm_graph_context(graph_params_for_mtp(params, mtp_model)), ++ target(target_model), ++ mtp(mtp_model) { ++ const int64_t n_bb = mtp.hparams.n_embd_out_impl; ++ GGML_ASSERT(n_bb > 0); ++ GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); ++ ++ // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 ++ // (fused path) chain from the previous step's on-device argmax + post-projection, so only ++ // step 0 reads token/h from the host. ++ ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); ++ ggml_set_input(inp_tok); ++ cb(inp_tok, "mtp_inp_last_token", -1); ++ ++ ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); ++ ggml_set_input(inp_h); ++ cb(inp_h, "mtp_inp_h_prev", -1); ++ ++ // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into ++ // ONE graph (no per-step host round-trip / GPU sync — the throughput fix). n_steps==1 is the ++ // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). ++ const int32_t n_steps = std::max(params.n_mtp_steps, 1); ++ ++ auto * inp_attn = build_attn_inp_kv_iswa(); ++ ++ if (n_steps <= 1) { ++ { ++ auto inp_wrap = std::make_unique(); ++ inp_wrap->inp_last_token = inp_tok; ++ inp_wrap->inp_h_prev = inp_h; ++ res->add_input(std::move(inp_wrap)); ++ } ++ ++ ggml_tensor * inp_pos = build_inp_pos(); ++ ++ ggml_tensor * logits = nullptr; ++ ggml_tensor * h_post = nullptr; ++ ggml_tensor * arg = nullptr; ++ gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, ++ inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); ++ ++ res->t_embd = h_post; ++ res->t_logits = logits; ++ res->t_argmax = arg; ++ ++ ggml_build_forward_expand(gf, arg); ++ ggml_build_forward_expand(gf, h_post); ++ return; ++ } ++ ++ // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is ++ // shared across steps — see gemma4_mtp_build_one_step's contract). The chain ties step k+1's ++ // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. ++ auto inp_wrap = std::make_unique(); ++ inp_wrap->inp_last_token = inp_tok; ++ inp_wrap->inp_h_prev = inp_h; ++ inp_wrap->inp_pos_steps.reserve(n_steps); ++ ++ std::vector pos_steps; ++ pos_steps.reserve(n_steps); ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); ++ ggml_set_input(p); ++ cb(p, "mtp_inp_pos_step", k); ++ pos_steps.push_back(p); ++ inp_wrap->inp_pos_steps.push_back(p); ++ } ++ res->add_input(std::move(inp_wrap)); ++ ++ ggml_tensor * tok_k = inp_tok; ++ ggml_tensor * h_k = inp_h; ++ ++ std::vector step_args; ++ step_args.reserve(n_steps); ++ ggml_tensor * last_logits = nullptr; ++ ggml_tensor * last_h_post = nullptr; ++ ++ for (int32_t k = 0; k < n_steps; ++k) { ++ ggml_tensor * logits_k = nullptr; ++ ggml_tensor * h_post_k = nullptr; ++ ggml_tensor * arg_k = nullptr; ++ gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, ++ tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); ++ ++ step_args.push_back(arg_k); ++ last_logits = logits_k; ++ last_h_post = h_post_k; ++ ++ // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the ++ // target vocab — already used as a get_rows index inside build_one_step), next step's ++ // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. ++ tok_k = arg_k; ++ h_k = h_post_k; ++ } ++ ++ // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 ++ // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend — a ++ // negligible side-consumer that does NOT touch the on-device get_rows chain above. ++ ggml_tensor * all_args = step_args[0]; ++ for (int32_t k = 1; k < n_steps; ++k) { ++ all_args = ggml_concat(ctx0, all_args, step_args[k], 0); ++ } ++ cb(all_args, "mtp_fused_argmax", -1); ++ ++ res->t_argmax = all_args; // I32 [n_steps] — the drafted token block ++ res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed ++ res->t_logits = last_logits; // unused on the live path (out_logits == NULL) ++ ++ ggml_build_forward_expand(gf, all_args); ++ ggml_build_forward_expand(gf, last_h_post); ++} ++ ++// opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via ++// llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path ++// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the ++// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 ++// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. ++void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { ++ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ++ ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); ++ ++ uint32_t n_kv_shared_layers = 0; ++ ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); ++ ++ hparams.n_layer_kv_from_start = hparams.n_layer - (int32_t) n_kv_shared_layers; ++ hparams.f_attention_scale = 1.0f; ++ ++ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); ++ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ++ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); ++ ++ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); ++ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); ++ ++ // opencoti F5 M6-S4 mtp: backbone width via the generic upstream key (#23398: %s.embedding_length_out ++ // → hparams.n_embd_out_impl), replacing our bespoke %s.n_embd_backbone. llama_model::load_hparams also ++ // reads it generically; re-read here so the assistant loader is self-contained. ++ ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_N_CENTROIDS, hparams.n_centroids, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_CENTROID_TOP_K, hparams.centroid_top_k, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_ATTENTION_K_EQ_V, hparams.attention_k_eq_v, false); ++ ml.get_key(LLM_KV_GEMMA4_ASSISTANT_USE_ORDERED_EMBEDDINGS, hparams.use_ordered_embeddings, false); ++ ++ type = LLM_TYPE_UNKNOWN; ++} ++ ++void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { ++ LLAMA_LOAD_LOCALS; ++ ++ const uint32_t n_bb = hparams.n_embd_out_impl; ++ if (n_bb == 0) { ++ throw std::runtime_error("gemma4-assistant: embedding_length_out (backbone width) must be set in GGUF metadata"); ++ } ++ if (n_embd_head_k != n_embd_head_v) { ++ throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k == n_embd_head_v"); ++ } ++ if (hparams.n_embd_head_k_swa != hparams.n_embd_head_v_swa) { ++ throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k_swa == n_embd_head_v_swa"); ++ } ++ ++ // opencoti F5 M6-S4 mtp (#408/bug-408): the assistant's token_embd is the TIED OUTPUT head (a dense ++ // LM head used only as a build_lora_mm weight, never an input get_rows — the input embedding comes ++ // from the TARGET's tok_embd). The P4 throughput win (#408) routes it to the OUTPUT buft so the ++ // per-step full-vocab mul_mat runs on the GPU instead of a graph-fragmenting CPU island. TOKEN_EMBD ++ // classifies LLM_TENSOR_LAYER_INPUT, so the standard create_tensor would land it on the CPU buft. ++ // create_tensor_output_head() (llama_model_base) passes dev_output.buft_list in the INPUT slot to ++ // force GPU output placement — additive, sole-use accounting (NOT TENSOR_DUPLICATED). This re-claims ++ // the 1.46× the 0.10.3 bump dropped; correctness is byte-identical (placement only). ++ tok_embd = create_tensor_output_head(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); ++ ++ // opencoti F5 M6-S4 mtp: GGUF tensor names upstream-aligned (#23398/#24282) — nextn.* / masked_embd_*. ++ // The C++ member names stay mtp_* (control-plane; our single-context engine, not upstream's ctx_other). ++ mtp_pre_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_PRE, "weight"), {2 * (int64_t) n_bb, n_embd}, 0); ++ mtp_post_projection = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_POST, "weight"), {n_embd, (int64_t) n_bb}, 0); ++ ++ if (hparams.use_ordered_embeddings) { ++ const uint32_t n_c = hparams.n_centroids; ++ if (n_c == 0) { ++ throw std::runtime_error("gemma4-assistant: use_ordered_embeddings requires n_centroids > 0"); ++ } ++ mtp_centroids = create_tensor(tn(LLM_TENSOR_MASKED_EMBD_CENTROIDS, "weight"), {n_embd, (int64_t) n_c}, 0); ++ // upstream loads ordering WITHOUT a .weight suffix → bare "masked_embd_ordering" ++ mtp_token_ordering = create_tensor(tn(LLM_TENSOR_MASKED_EMBD_ORDERING), {(int64_t) n_vocab}, TENSOR_NOT_REQUIRED); ++ } ++ ++ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); ++ ++ int rope_freqs_flag = 0; ++ ++ for (int i = 0; i < n_layer; ++i) { ++ auto & layer = layers[i]; ++ const int64_t n_head = hparams.n_head(i); ++ const int64_t n_embd_head = hparams.n_embd_head_k(i); ++ ++ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); ++ ++ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); ++ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0); ++ ++ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0); ++ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); ++ ++ layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), {1u}, TENSOR_NOT_REQUIRED); ++ ++ if (!hparams.is_swa(i)) { ++ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head/2}, rope_freqs_flag); ++ rope_freqs_flag = TENSOR_DUPLICATED; ++ } ++ ++ const int64_t n_ff_cur = hparams.n_ff(i); ++ ++ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); ++ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff_cur}, 0); ++ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_cur}, 0); ++ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_cur, n_embd}, 0); ++ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); ++ } ++} ++ ++std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params &) const { ++ throw std::runtime_error( ++ "gemma4_assistant cannot be used as a primary model (-m). " ++ "Load the Gemma 4 target with -m, then call llama_model_load_mtp_from_file() with the assistant GGUF."); ++}