From: opencoti Subject: [PATCH 0096] remove single-context Gemma MTP facade; dual-ctx is the only assistant path (#611) END of the bug-858 port: delete the retired single-context in-target assistant engine (~1600 lines) now that the dual-context port (0093) proved accept parity: - llama-context: decode_mtp facade + decode_mtp_sync/fused, async worker (decode_mtp_async/wait/run, mtp_worker thread, drain guard), process_ubatch_mtp, graph_params_mtp, ensure_sched_mtp, sched_mtp/gf_res_prev_mtp, graph_compute_mtp. - public API llama_decode_mtp removed (llama_decode_mtp_fused_nextn stays). - llama-graph: build_attn_mtp (iswa cross-read); kv-cache-iswa::init_mtp. - models: llm_build_gemma4_mtp + one-step builder + gemma4.cpp MTP dispatch; gemma4-assistant keeps ONLY the dual-ctx graph (ctx_other required). - speculative: common_speculative_impl_draft_assistant + NDJSON tracer removed; --spec-type draft-assistant now requires ctx_dft (server always creates it). - server: OPENCOTI_MTP_DUAL_CTX env gate removed — dual-ctx unconditional; the --parallel>1 -> --kv-unified boot guard is preserved (shared-KV aliasing has the same per-stream-split constraint the facade had). NextN side untouched (mtp_fused_steps, sched_nextn, mtp_slot_info, unified twin builders); mtp_assistant model loading kept. Gate on bs2: A4B n2 305.4 tps / 0.849 accept (== 0093 reference, boot line proves dual-ctx default), q35 n3 248.8 / 0.662. bug-2110 (par2 accept 0.635 on the tps harness) was RESOLVED as a harness artifact: on the real 9-prompt mtp-bench the dual-ctx assistant holds accept at --parallel 2 --kv-unified (ours 0.785 par1 / 0.783 par2; upstream b9859 0.793 / 0.792). diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h index 3bb6aa2..887dfb6 100644 --- a/llama.cpp/common/common.h +++ b/llama.cpp/common/common.h @@ -305,7 +305,7 @@ struct common_params_speculative_draft { // opencoti F5 M6-S4 mtp — MTP (Gemma 4 assistant): OPTIONAL explicit draft-depth ceiling. // 0 (default) = draft depth follows --spec-draft-n-max (n_max); the MTP head re-runs - // autoregressively per step (decode_mtp_sync/fused), so depth is NOT limited to a fixed + // autoregressively per step (draft_mtp / decode_mtp_fused_nextn), so depth is NOT limited to a fixed // block size. bug-858: the old default 3 hard-capped ours at 2 drafts/round and blocked // n-max scaling vs upstream b9859. When set >1, caps draft depth at draft_block_size-1. int32_t draft_block_size = 0; diff --git a/llama.cpp/common/speculative.cpp b/llama.cpp/common/speculative.cpp index ee7962f..010bf65 100644 --- a/llama.cpp/common/speculative.cpp +++ b/llama.cpp/common/speculative.cpp @@ -1409,315 +1409,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { } }; -// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. -// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, -// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV -// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens -// of a block in ONE facade call (no per-step sampler loop). -// -// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), -// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never -// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() -// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) -// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). - -// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. -// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. -namespace { -struct mtp_acc_tracer { - bool enabled = false; - FILE * fp = nullptr; - std::mutex mu; - - mtp_acc_tracer() { - const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); - if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { - return; - } - fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); - enabled = (fp != nullptr); - } - - ~mtp_acc_tracer() { - if (fp && fp != stderr) { - std::fclose(fp); - } - } - - void writeln(const std::string & line) { - if (!enabled) { - return; - } - std::lock_guard lk(mu); - std::fputs(line.c_str(), fp); - std::fputc('\n', fp); - std::fflush(fp); - } -}; - -mtp_acc_tracer & mtp_tracer() { - static mtp_acc_tracer t; - return t; -} -} // namespace - -struct common_speculative_impl_draft_assistant : public common_speculative_impl { - common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) - - llama_context * ctx_tgt = nullptr; - - int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes - int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row - - // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows - // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds - // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h - // scheme so multi-seq (PolyKV) works without per-seq server plumbing. - std::vector> pending_h; // [n_seq][n_embd_backbone] - std::vector> verify_h; // [n_seq][n_rows * n_embd_backbone] - std::vector verify_h_rows; - std::vector i_batch_beg; - std::vector i_batch_end; - - // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head - // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain - // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. - int skip_streak_threshold = 0; - std::vector zero_accept_streak; - std::vector skip_last_draft; - std::vector prev_n_acc_at_draft; - - int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change - - static float compute_h_l2(const float * h, int32_t n) { - if (!h || n <= 0) { - return 0.0f; - } - double s = 0.0; - for (int32_t i = 0; i < n; ++i) { - s += (double) h[i] * (double) h[i]; - } - return (float) std::sqrt(s); - } - - common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) - , params(params.draft) - { - ctx_tgt = this->params.ctx_tgt; - GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); - - const llama_model * model_tgt = llama_get_model(ctx_tgt); - n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); - n_embd_out = llama_model_n_embd_out(model_tgt); - - LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); - LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, - this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); - - // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. - llama_set_embeddings(ctx_tgt, true); - - if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { - const int v = std::atoi(e); - if (v >= 1 && v <= 32) { - skip_streak_threshold = v; - } - } - - pending_h.assign(n_seq, std::vector((size_t) std::max(1, n_embd_backbone), 0.0f)); - verify_h.assign(n_seq, {}); - verify_h_rows.assign(n_seq, 0); - i_batch_beg.assign(n_seq, -1); - i_batch_end.assign(n_seq, -1); - zero_accept_streak.assign(n_seq, 0); - skip_last_draft.assign(n_seq, 0); - prev_n_acc_at_draft.assign(n_seq, 0); - } - - ~common_speculative_impl_draft_assistant() override = default; - - void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { - llama_set_embeddings(ctx_tgt, true); - if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { - skip_last_draft[seq_id] = 0; - zero_accept_streak[seq_id] = 0; - } - } - - // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() - // can select the last-accepted token's hidden state. The spec framework calls process() after the - // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which - // mirrors into a separate ctx_dft) — the assistant reads the target's own embeddings directly. - // NOTE (bug-858): PRE-norm harvest was TESTED and REFUTED — it collapses accept to ~0.05 at ALL - // positions (incl short ctx). This drafter expects the post-norm hidden; do NOT swap to pre_norm. - bool process(const llama_batch & batch_in) override { - if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { - return true; - } - if (n_embd_backbone <= 0) { - return true; - } - - const int32_t n_tokens = batch_in.n_tokens; - std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); - std::fill(i_batch_end.begin(), i_batch_end.end(), -1); - - for (int k = 0; k < n_tokens; ++k) { - const llama_seq_id s = batch_in.seq_id[k][0]; - if (s < 0 || s >= (llama_seq_id) n_seq) { - continue; - } - i_batch_end[s] = k; - if (i_batch_beg[s] < 0) { - i_batch_beg[s] = k; - } - } - - const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); - const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); - - for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { - if (i_batch_beg[s] < 0) { - continue; - } - const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; - verify_h_rows[s] = n_rows; - verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); - - for (int32_t i = 0; i < n_rows; ++i) { - const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); - if (h) { - std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, - (size_t) n_copy * sizeof(float)); - } - } - // default pending_h to the last row (correct when all drafts accepted / on prefill) - std::memcpy(pending_h[s].data(), - verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); - } - - return true; - } - - void draft(common_speculative_draft_params_vec & dparams) override { - if (n_embd_backbone <= 0) { - return; - } - - // bug-858: draft depth follows --spec-draft-n-max (n_max) and the room left in context - // (dp.n_max = n_draft_max), NOT draft_block_size. The MTP head re-runs autoregressively per - // step (decode_mtp_sync/fused build a fresh single-step graph each step: argmax -> next - // last_token, h_post -> next h_prev), so depth is bounded by n_max, not a fixed block. The - // legacy draft_block_size-1 ceiling (default 3 -> 2) hard-capped ours at 2 drafts/round and - // blocked n-max scaling vs upstream b9859 (which drafts to n_max autoregressively) -> REMOVED. - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, - // the previous batch produced 0 accepted drafts. - if (skip_last_draft[seq_id]) { - skip_last_draft[seq_id] = 0; - } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { - zero_accept_streak[seq_id]++; - } else { - zero_accept_streak[seq_id] = 0; - } - if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { - zero_accept_streak[seq_id] = 0; - skip_last_draft[seq_id] = 1; - prev_n_acc_at_draft[seq_id] = n_acc_drafts; - continue; // empty result -> server falls back to single-token verify - } - - int32_t n_steps = params.n_max > 0 ? params.n_max : 1; - if (dp.n_max > 0) { - n_steps = std::min(n_steps, dp.n_max); - } - if (n_steps <= 0) { - prev_n_acc_at_draft[seq_id] = n_acc_drafts; - continue; - } - - float * h_prev = pending_h[seq_id].data(); - - llama_memory_t mem = llama_get_memory(ctx_tgt); - llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; - if (attn_pos < 0) { - attn_pos = 0; - } - - std::vector out((size_t) n_steps, 0); - const int32_t rc = llama_decode_mtp( - ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, - /*out_drafts =*/ out.data(), - /*out_logits =*/ nullptr, - /*out_h_prev_last=*/ nullptr); - - prev_n_acc_at_draft[seq_id] = n_acc_drafts; - - if (rc != 0) { - LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); - continue; - } - - auto & result = *dp.result; - for (int32_t i = 0; i < n_steps; ++i) { - result.push_back(out[i]); - } - - if (mtp_tracer().enabled) { - std::ostringstream oss; - oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter - << ",\"seq_id\":" << (int) seq_id - << ",\"id_last\":" << (int) dp.id_last - << ",\"attn_pos\":" << (int) attn_pos - << ",\"n_steps\":" << n_steps - << ",\"h_l2\":" << std::fixed << std::setprecision(4) - << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; - for (int32_t i = 0; i < n_steps; ++i) { - if (i) oss << ','; - oss << (int) out[i]; - } - oss << "]}"; - mtp_tracer().writeln(oss.str()); - } - } - } - - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - return; - } - const int32_t n_rows = verify_h_rows[seq_id]; - if (n_rows <= 0 || n_embd_backbone <= 0) { - return; - } - // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the - // last-accepted token's hidden state (clamped to the harvested rows). - const int32_t i_h = std::min(n_accepted, n_rows - 1); - std::memcpy(pending_h[seq_id].data(), - verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, - (size_t) n_embd_backbone * sizeof(float)); - - if (mtp_tracer().enabled) { - std::ostringstream oss; - oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter - << ",\"seq_id\":" << (int) seq_id - << ",\"n_accepted\":" << (int) n_accepted << "}"; - mtp_tracer().writeln(oss.str()); - ++trace_iter; - } - } - - bool need_embd() const override { - return true; // MTP reads POST-norm output embeddings (h_prev) - } -}; struct common_speculative { common_speculative_draft_params_vec dparams; @@ -1934,15 +1625,15 @@ common_speculative * common_speculative_init(common_params_speculative & params, break; } case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter - // opencoti bug-858 dual-context MTP: when the server created a real ctx_dft FROM the - // assistant model (ctx_other=ctx_tgt; --spec-type draft-assistant + OPENCOTI_MTP_DUAL_CTX), - // run the shared-KV draft_mtp driver (upstream b9859 is_mem_shared). Otherwise fall - // back to the proven single-context in-target assistant (ctx_dft == nullptr). - if (config.params.draft.ctx_dft != nullptr) { - impls.push_back(std::make_unique(config.params, n_seq)); - } else { - impls.push_back(std::make_unique(config.params, n_seq)); + // opencoti bug-858 dual-context MTP (#611): the assistant ALWAYS runs as a real + // ctx_dft created FROM the assistant model (ctx_other=ctx_tgt) through the shared-KV + // draft_mtp driver (upstream b9859 is_mem_shared). The single-context in-target + // facade was removed in #611 after the dual-context port proved accept parity. + if (config.params.draft.ctx_dft == nullptr) { + LOG_ERR("%s: draft-assistant requires the dual-context draft (server creates ctx_dft with ctx_other=target)\n", __func__); + return nullptr; } + impls.push_back(std::make_unique(config.params, n_seq)); break; } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { @@ -2010,8 +1701,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, // opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / // set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative // framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ -// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on -// common_speculative_impl_draft_assistant above). +// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete. void common_speculative_free(common_speculative * spec) { if (spec == nullptr) { diff --git a/llama.cpp/include/llama.h b/llama.cpp/include/llama.h index e6517be..2f4da86 100644 --- a/llama.cpp/include/llama.h +++ b/llama.cpp/include/llama.h @@ -1041,22 +1041,6 @@ extern "C" { struct llama_context * ctx, struct llama_batch batch); - // opencoti F5 M6-S4 mtp - // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, - // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). - // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) - // receive per-step logits and the final backbone hidden. Returns 0 on success. - LLAMA_API int32_t llama_decode_mtp( - struct llama_context * ctx, - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last); - // opencoti fused-NextN (#590): fuse N per-step NextN draft decodes into one graph on a // DECODER_MTP (Qwen NextN) draft context. GREEDY-ONLY (no p_min early-stop). Returns 0 on // success. last_token must already be resident in ctx's cache at attn_pos (seed decode first). diff --git a/llama.cpp/src/llama-arch.h b/llama.cpp/src/llama-arch.h index 9aef864..85b1e98 100644 --- a/llama.cpp/src/llama-arch.h +++ b/llama.cpp/src/llama-arch.h @@ -570,8 +570,8 @@ enum llm_tensor { // opencoti F5 M6-S4 mtp — Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile - // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our - // single-context build_attn_mtp cross-read — NOT upstream's ctx_other. See plan. + // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine is the + // bug-858 dual-context port (cparams.ctx_other, upstream-shaped). See plan. LLM_TENSOR_NEXTN_PROJ_PRE, LLM_TENSOR_NEXTN_PROJ_POST, LLM_TENSOR_MASKED_EMBD_CENTROIDS, diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp index 7d27212..1579acc 100644 --- a/llama.cpp/src/llama-context.cpp +++ b/llama.cpp/src/llama-context.cpp @@ -9,7 +9,7 @@ #include "llama-memory.h" #include "llama-mmap.h" #include "llama-model.h" -#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp +#include "llama-kv-cache-iswa.h" #include "llama-ext.h" #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard #include "llama.h" @@ -527,17 +527,6 @@ llama_context::llama_context( } llama_context::~llama_context() { - // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down - // sched_mtp and the backends it touches. - if (mtp_worker.joinable()) { - { - std::lock_guard lk(mtp_mu); - mtp_worker_stop.store(true, std::memory_order_release); - } - mtp_cv_request.notify_all(); - mtp_worker.join(); - } - if (!model.hparams.no_alloc) { for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; @@ -1421,208 +1410,6 @@ bool llama_context::set_adapter_cvec( return res; } -// opencoti F5 M6-S4 mtp (P2 inert) -bool llama_context::ensure_sched_mtp() { - // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count - // (mtp_fused_steps). A fused N-step graph has ~N× the nodes of the single-step graph and a - // distinct topology, so when a larger block is requested we drop and rebuild the reserve. - const int32_t need = std::max(mtp_fused_steps, 1); - if (sched_mtp && mtp_reserved_steps >= need) { - return true; - } - if (!model.mtp_assistant) { - return false; - } - sched_mtp.reset(); - gf_res_prev_mtp.reset(); - - const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; - - gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); - sched_mtp.reset(ggml_backend_sched_new( - backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), - max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); - if (!sched_mtp) { - LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); - gf_res_prev_mtp.reset(); - return false; - } - - // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. - // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), - // so a single reserve covers all subsequent decode_mtp calls. - { - auto * kv_iswa = dynamic_cast(memory.get()); - if (!kv_iswa) { - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); - sched_mtp.reset(); - gf_res_prev_mtp.reset(); - return false; - } - - llama_memory_context_ptr mctx = memory->init_full(); - if (!mctx) { - LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); - sched_mtp.reset(); - gf_res_prev_mtp.reset(); - return false; - } - - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; - auto data = std::make_shared(); - data->token.resize(1); - data->embd.resize(n_bb); - data->pos.resize(1); - data->n_seq_id.resize(1); - data->seq_id.resize(1); - data->seq_id_data.resize(1); - data->output.resize(1); - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); - data->seq_id_unq.push_back(0); - data->seq_idx[0] = 0; - data->n_seq_id[0] = 1; - data->seq_id_data[0] = 0; - data->seq_id[0] = &data->seq_id_data[0]; - data->output[0] = 1; - - llama_ubatch ub{}; - ub.b_equal_seqs = 1; - ub.n_tokens = 1; - ub.n_seq_tokens = 1; - ub.n_seqs = 1; - ub.n_seqs_unq = 1; - ub.n_pos = 1; - ub.token = data->token.data(); - ub.embd = data->embd.data(); - ub.pos = data->pos.data(); - ub.n_seq_id = data->n_seq_id.data(); - ub.seq_id = data->seq_id.data(); - ub.seq_id_unq = data->seq_id_unq.data(); - ub.seq_idx = data->seq_idx.data(); - ub.output = data->output.data(); - ub.data = data; - - const uint32_t save_n_outputs = n_outputs; - n_outputs = 1; - - auto * res = gf_res_prev_mtp.get(); - const auto gparams = graph_params_mtp(res, ub, mctx.get()); - res->reset(); - ggml_backend_sched_reset(sched_mtp.get()); - - auto * gf = model.build_graph(gparams); - n_outputs = save_n_outputs; - - if (!gf) { - LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); - sched_mtp.reset(); - gf_res_prev_mtp.reset(); - return false; - } - if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { - LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); - sched_mtp.reset(); - gf_res_prev_mtp.reset(); - return false; - } - // Discard the reserve graph cache so the first real call rebuilds with proper inputs. - res->reset(); - ggml_backend_sched_reset(sched_mtp.get()); - } - - // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in - // decode_mtp_async (only when the opt-in async path is actually taken), NOT here — so the - // default in-thread sync path creates no extra thread and is runtime-identical to P3. - - mtp_reserved_steps = need; // reserve now covers `need` fused steps - return true; -} - -// opencoti F5 M6-S4 mtp (P2 inert) -llm_graph_result * llama_context::process_ubatch_mtp( - const llama_ubatch & ubatch, - llama_memory_context_i * mctx, - ggml_status & ret) { - GGML_ASSERT(sched_mtp && gf_res_prev_mtp); - - auto * res = gf_res_prev_mtp.get(); - auto * gf = res->get_gf(); - - // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it - // explicitly here in case a concurrent target decode mutates it. The MTP graph - // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. - llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); - gparams.n_outputs = 1; - gparams.sched = sched_mtp.get(); - - if (!graph_reuse_disable && res->can_reuse(gparams)) { - // sched_mtp does not use pipeline parallelism (created with pipeline=false), - // so no synchronize is needed before set_inputs. - } else { - res->reset(); - - ggml_backend_sched_reset(sched_mtp.get()); - ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); - - gf = model.build_graph(gparams); - if (!gf) { - LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); - ret = GGML_STATUS_FAILED; - return nullptr; - } - if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { - LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); - ret = GGML_STATUS_ALLOC_FAILED; - return nullptr; - } - } - - res->set_inputs(&ubatch); - - const auto status = graph_compute_mtp(res->get_gf()); - - if (status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); - ret = status; - return nullptr; - } - - ret = GGML_STATUS_SUCCESS; - return res; -} - -// opencoti F5 M6-S4 mtp (P2 inert) -ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { - // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s - // threadpool/n_threads dance only for the CPU backend; the MTP head is small and - // thread saturation matters less, but consistency with graph_compute() avoids - // surprising backend reconfigurations between target and MTP runs. - // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker - // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. - const int n_threads = cparams.n_threads; - ggml_threadpool_t tp = threadpool; - - if (backend_cpu != nullptr) { - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); - auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) - ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); - if (set_threadpool_fn) { - set_threadpool_fn(backend_cpu, tp); - } - } - for (const auto & set_n_threads_fn : set_n_threads_fns) { - set_n_threads_fn.second(set_n_threads_fn.first, n_threads); - } - - auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); - if (status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", - __func__, status); - } - return status; -} - llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { if (mctx && !mctx->apply()) { LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); @@ -2093,14 +1880,6 @@ int llama_context::decode(const llama_batch & batch_inp) { bool did_optimize = false; - // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its - // KV below (memory_update's pending shifts/copies, then init_batch's slot apply — the S1 - // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this - // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in - // the current sync-facade integration (nothing is in flight here) and a single pointer - // check when no MTP assistant is loaded. - mtp_drain_before_mutate(); - // handle any pending shifts/copies memory_update(false); @@ -2684,368 +2463,11 @@ ggml_cgraph * llama_context::graph_reserve( return gf; } -// opencoti F5 M6-S4 mtp (P2 inert) -llm_graph_params llama_context::graph_params_mtp( - llm_graph_result * res, - const llama_ubatch & ubatch, - const llama_memory_context_i * mctx) const { - const llama_model * mtp = model.mtp_assistant.get(); - GGML_ASSERT(mtp); - - return { - /*.arch =*/ mtp->arch, - /*.hparams =*/ mtp->hparams, - /*.cparams =*/ cparams, - /*.ubatch =*/ ubatch, - /*.gtype =*/ LLM_GRAPH_TYPE_MTP, - /*.sched =*/ sched.get(), - /*.backend_cpu =*/ backend_cpu, - /*.cvec =*/ cvec.get(), - /*.loras =*/ loras.get(), - /*.mctx =*/ mctx, - /*.cross =*/ &cross, - /*.samplers =*/ sampling.samplers, - /*.n_outputs =*/ n_outputs, - /*.cb =*/ graph_get_cb(), - /*.res =*/ res, - /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) - }; -} - -// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. -int32_t llama_context::decode_mtp( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last) { - // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams - // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the - // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes async→wait. - // With no target/draft overlap this is byte-identical to the P3 sync path — the worker runs - // the same N-step loop off-thread while we block in decode_mtp_wait. - if (out_logits) { - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, - out_drafts, out_logits, out_h_prev_last); - } - // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the - // fused single-graph draft (decode_mtp_fused) — ONE launch+sync+D2H for the whole block, the - // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused - // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse → sequential. - static const bool fused_enabled = [] { - const char * e = getenv("OPENCOTI_MTP_FUSED"); - return !(e && e[0] == '0'); - }(); - if (fused_enabled && n_steps > 1) { - return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, - out_drafts, out_h_prev_last); - } - // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). - // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker - // gives NO throughput benefit — real depth-2 overlap is deferred (limited by the h-prev - // dependency; even the AtomicBot fork ships depth-1) — AND the worker-thread CUDA execution - // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off - // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. - static const bool async_enabled = [] { - const char * e = getenv("OPENCOTI_MTP_ASYNC"); - return e && e[0] == '1'; - }(); - if (!async_enabled) { - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, - out_drafts, out_logits, out_h_prev_last); - } - const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); - if (rc != 0) { - return rc; - } - return decode_mtp_wait(out_drafts, out_h_prev_last); -} - -// opencoti F5 M6-S4 mtp (P2 inert) -int32_t llama_context::decode_mtp_sync( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last) { - if (!model.mtp_assistant) { - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); - return -1; - } - if (!memory) { - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); - return -2; - } - auto * kv_iswa = dynamic_cast(memory.get()); - if (!kv_iswa) { - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); - return -3; - } - - // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, - // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) - mtp_fused_steps = 1; - - if (!ensure_sched_mtp()) { - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); - return -8; - } - - const int32_t n_vocab = model.vocab.n_tokens(); - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; - if (n_bb == 0) { - LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); - return -4; - } - - auto data = std::make_shared(); - data->token.resize(1); - data->embd.resize(n_bb); - data->pos.resize(1); - data->n_seq_id.resize(1); - data->seq_id.resize(1); - data->seq_id_data.resize(1); - data->output.resize(1); - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); - data->seq_id_unq.push_back(seq_id); - data->seq_idx[(size_t) seq_id] = 0; - - llama_ubatch ub{}; - ub.b_equal_seqs = 1; - ub.n_tokens = 1; - ub.n_seq_tokens = 1; - ub.n_seqs = 1; - ub.n_seqs_unq = 1; - ub.n_pos = 1; - ub.token = data->token.data(); - ub.embd = data->embd.data(); - ub.pos = data->pos.data(); - ub.n_seq_id = data->n_seq_id.data(); - ub.seq_id = data->seq_id.data(); - ub.seq_id_unq = data->seq_id_unq.data(); - ub.seq_idx = data->seq_idx.data(); - ub.output = data->output.data(); - ub.data = data; - - data->n_seq_id[0] = 1; - data->seq_id_data[0] = seq_id; - data->seq_id[0] = &data->seq_id_data[0]; - // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post - // back). It MUST be flagged output=1 so build_inp_out_ids selects its row — and so the live - // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here - // produced a divergent graph whose output-selection tensor was left with a null backend - // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. - data->output[0] = 1; - - if (n_steps <= 0) { - return 0; - } - - // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token - // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. - for (int32_t k = 0; k < n_steps; ++k) { - data->token[0] = last_token; - // bug-858: the gemma4-assistant (is_mem_shared) drafts ALL steps from the SAME query - // position — the head is trained to predict multiple future tokens from one position, with - // the recurrence carried by the h_prev hidden chain (NOT the RoPE angle). Incrementing the - // position per step (attn_pos+1+k) hands the head a mistrained RoPE angle from step 1 on, - // degrading step-1+ drafts (aggregate accept ~0.69 vs upstream 0.79). Match b9859 draft_mtp - // is_mem_shared: common_batch_add(batch, id, dp.n_past, ...) — constant dp.n_past = attn_pos+1. - data->pos[0] = attn_pos + 1; - std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); - - llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); - return -5; - } - - ggml_status status = GGML_STATUS_SUCCESS; - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); - if (!res || status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); - return -6; - } - - ggml_backend_sched_synchronize(sched_mtp.get()); - - ggml_tensor * t_arg = res->get_argmax(); - GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); - - int32_t best_i32 = 0; - ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); - out_drafts[k] = (llama_token) best_i32; - - if (out_logits) { - ggml_tensor * t_logits = res->get_logits(); - GGML_ASSERT(t_logits); - ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, - 0, (size_t) n_vocab * sizeof(float)); - } - - ggml_tensor * t_post = res->get_embd(); - GGML_ASSERT(t_post); - ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); - - last_token = (llama_token) best_i32; - } - - if (out_h_prev_last) { - std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); - } - - return 0; -} - -// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head -// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls -// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index — already -// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. -// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, -// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). -// -// One mask shared across steps (built from step 0's position). For FULL-attn target layers and -// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa — the short-context / -// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are -// byte-identical to the sequential path. For SWA layers at long context the true per-step window -// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is -// output-safe — the target verify pass rejects any mismatched draft, so the emitted stream is -// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix -// is a per-step mask slice — deferred until measured. See buglog bug-420.) -int32_t llama_context::decode_mtp_fused( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_h_prev_last) { - if (!model.mtp_assistant) { - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); - return -1; - } - if (!memory) { - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); - return -2; - } - auto * kv_iswa = dynamic_cast(memory.get()); - if (!kv_iswa) { - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); - return -3; - } - if (n_steps <= 0) { - return 0; - } - // A single step has nothing to fuse — defer to the proven sequential path. - if (n_steps == 1) { - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, - out_drafts, /*out_logits=*/nullptr, out_h_prev_last); - } - - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; - if (n_bb == 0) { - LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); - return -4; - } - - // Build/reserve a fused n_steps graph. - mtp_fused_steps = n_steps; - if (!ensure_sched_mtp()) { - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); - return -8; - } - - // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step - // query positions consumed by the wrapper's inp_pos_steps (set_input). - auto data = std::make_shared(); - data->token.resize(1); - data->embd.resize(n_bb); - data->pos.resize(n_steps); - data->n_seq_id.resize(1); - data->seq_id.resize(1); - data->seq_id_data.resize(1); - data->output.resize(1); - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); - data->seq_id_unq.push_back(seq_id); - data->seq_idx[(size_t) seq_id] = 0; - - llama_ubatch ub{}; - ub.b_equal_seqs = 1; - ub.n_tokens = 1; - ub.n_seq_tokens = 1; - ub.n_seqs = 1; - ub.n_seqs_unq = 1; - ub.n_pos = (uint32_t) n_steps; - ub.token = data->token.data(); - ub.embd = data->embd.data(); - ub.pos = data->pos.data(); - ub.n_seq_id = data->n_seq_id.data(); - ub.seq_id = data->seq_id.data(); - ub.seq_id_unq = data->seq_id_unq.data(); - ub.seq_idx = data->seq_idx.data(); - ub.output = data->output.data(); - ub.data = data; - - data->n_seq_id[0] = 1; - data->seq_id_data[0] = seq_id; - data->seq_id[0] = &data->seq_id_data[0]; - data->output[0] = 1; - - data->token[0] = last_token; - std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); - // bug-858: constant query position for ALL draft steps (see decode_mtp_sync note). The - // gemma4-assistant (is_mem_shared) is trained to predict every future draft token from the - // SAME position (dp.n_past = attn_pos+1); recurrence lives in the h_prev chain, not the RoPE - // angle. The old attn_pos+1+k mistrained step-1+ RoPE -> accept 0.69 vs upstream 0.79. - for (int32_t k = 0; k < n_steps; ++k) { - data->pos[k] = attn_pos + 1; - } - - llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); - return -5; - } - - ggml_status status = GGML_STATUS_SUCCESS; - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); - if (!res || status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); - return -6; - } - - ggml_backend_sched_synchronize(sched_mtp.get()); - - ggml_tensor * t_arg = res->get_argmax(); - GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); - GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); - - std::vector drafts((size_t) n_steps); - ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); - for (int32_t k = 0; k < n_steps; ++k) { - out_drafts[k] = (llama_token) drafts[(size_t) k]; - } - - if (out_h_prev_last) { - ggml_tensor * t_post = res->get_embd(); - GGML_ASSERT(t_post); - ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); - } - - return 0; -} - // opencoti fused-NextN (#590 / bug-858): fuse the N per-step NextN draft decodes into ONE graph on // the draft context (ctx_dft, type LLAMA_CONTEXT_TYPE_MTP → LLM_GRAPH_TYPE_DECODER_MTP). This is the // throughput fix: the un-fused path pays a host sync per draft step; here one graph emits N greedy -// drafts with a single sync. Twin of decode_mtp_fused (Gemma assistant) but for the UNIFIED cache — -// so no mtp_assistant / sched_mtp / kv_iswa. The N fused steps read-only cross-attend the frozen +// drafts with a single sync, for the UNIFIED cache — +// so no mtp_assistant / kv_iswa involvement. The N fused steps read-only cross-attend the frozen // prefix (build_attn_readonly_nextn, Option A); recurrence is carried by the on-device hidden chain. // GREEDY-ONLY: no p_min early-stop (the caller reconciles draft length). See // docs/features/fused_nextn_mtp.md. Returns 0 on success, negative on error. @@ -3062,7 +2484,7 @@ int32_t llama_context::decode_mtp_fused_nextn( return -2; } // NextN self-spec targets are full-attention → unified cache. (iSWA NextN is not a thing; - // the Gemma assistant path is decode_mtp_fused.) + // the Gemma assistant runs as a dual context via draft_mtp, not through this driver.) auto * kv = dynamic_cast(memory.get()); if (!kv) { LLAMA_LOG_ERROR("%s: fused NextN requires a unified llama_kv_cache\n", __func__); @@ -3133,7 +2555,7 @@ int32_t llama_context::decode_mtp_fused_nextn( } // Build/compute the fused N-step graph on the NORMAL sched (graph_params sets n_mtp_steps=N for - // DECODER_MTP). Inlined like process_ubatch_mtp — deliberately NOT process_ubatch, whose apply() + // DECODER_MTP). Inlined graph processing — deliberately NOT process_ubatch, whose apply() // would overwrite the pmax prefix cell. mtp_fused_steps = n_steps; @@ -3187,7 +2609,7 @@ int32_t llama_context::decode_mtp_fused_nextn( } res->set_inputs(&ub); - // Compute on the dedicated sched (mirror graph_compute_mtp's CPU-threadpool dance). + // Compute on the dedicated sched (with the CPU-threadpool attach/set dance). if (backend_cpu != nullptr) { auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); auto * set_tp_fn = (decltype(ggml_backend_cpu_set_threadpool) *) @@ -3227,224 +2649,6 @@ int32_t llama_context::decode_mtp_fused_nextn( return 0; } -// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one -// in-flight request per context (returns -7 if a prior request was not yet waited). -int32_t llama_context::decode_mtp_async( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - const float * h_prev, - int32_t n_steps) { - if (!model.mtp_assistant) { - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); - return -1; - } - if (!memory) { - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); - return -2; - } - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; - if (n_bb == 0 || !h_prev || n_steps <= 0) { - LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", - __func__, n_bb, (const void *) h_prev, n_steps); - return -4; - } - if (!ensure_sched_mtp()) { - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); - return -8; - } - if (!mtp_worker.joinable()) { - mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); - } - - { - std::unique_lock lk(mtp_mu); - if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { - LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", - __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, - (int) mtp_completed.has_value()); - return -7; - } - mtp_request req; - req.seq_id = seq_id; - req.attn_pos = attn_pos; - req.last_token = last_token; - req.n_steps = n_steps; - req.h_prev.assign(h_prev, h_prev + n_bb); - mtp_pending = std::move(req); - } - mtp_cv_request.notify_one(); - return 0; -} - -// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last -// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). -int32_t llama_context::decode_mtp_wait( - llama_token * out_drafts, - float * out_h_prev_last) { - std::unique_lock lk(mtp_mu); - mtp_cv_response.wait(lk, [this] { - return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); - }); - if (!mtp_completed.has_value()) { - LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); - return -7; - } - mtp_response resp = std::move(*mtp_completed); - mtp_completed.reset(); - lk.unlock(); - - if (resp.status != 0) { - return resp.status; - } - if (out_drafts && !resp.drafts.empty()) { - std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); - } - if (out_h_prev_last && !resp.h_prev_last.empty()) { - std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); - } - return 0; -} - -// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's -// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the -// single ubatch row MUST be flagged output — a 0 here reintroduces the null-out-ids abort -// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. -int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { - auto * kv_iswa = dynamic_cast(memory.get()); - if (!kv_iswa) { - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); - return -3; - } - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; - - auto data = std::make_shared(); - data->token.resize(1); - data->embd.resize(n_bb); - data->pos.resize(1); - data->n_seq_id.resize(1); - data->seq_id.resize(1); - data->seq_id_data.resize(1); - data->output.resize(1); - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); - data->seq_id_unq.push_back(req.seq_id); - data->seq_idx[(size_t) req.seq_id] = 0; - - llama_ubatch ub{}; - ub.b_equal_seqs = 1; - ub.n_tokens = 1; - ub.n_seq_tokens = 1; - ub.n_seqs = 1; - ub.n_seqs_unq = 1; - ub.n_pos = 1; - ub.token = data->token.data(); - ub.embd = data->embd.data(); - ub.pos = data->pos.data(); - ub.n_seq_id = data->n_seq_id.data(); - ub.seq_id = data->seq_id.data(); - ub.seq_id_unq = data->seq_id_unq.data(); - ub.seq_idx = data->seq_idx.data(); - ub.output = data->output.data(); - ub.data = data; - - data->n_seq_id[0] = 1; - data->seq_id_data[0] = req.seq_id; - data->seq_id[0] = &data->seq_id_data[0]; - data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 - - std::vector h(req.h_prev); - llama_token last_token = req.last_token; - resp.drafts.assign((size_t) req.n_steps, 0); - - for (int32_t k = 0; k < req.n_steps; ++k) { - data->token[0] = last_token; - data->pos[0] = req.attn_pos + 1 + (llama_pos) k; - std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); - - llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); - return -5; - } - - ggml_status status = GGML_STATUS_SUCCESS; - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); - if (!res || status != GGML_STATUS_SUCCESS) { - LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); - return -6; - } - - ggml_backend_sched_synchronize(sched_mtp.get()); - - ggml_tensor * t_arg = res->get_argmax(); - GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); - int32_t best_i32 = 0; - ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); - last_token = (llama_token) best_i32; - resp.drafts[(size_t) k] = last_token; - - ggml_tensor * t_post = res->get_embd(); - GGML_ASSERT(t_post); - ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); - } - - resp.h_prev_last = std::move(h); - return 0; -} - -// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on -// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. -void llama_context::mtp_worker_loop() { - for (;;) { - mtp_request req; - { - std::unique_lock lk(mtp_mu); - mtp_cv_request.wait(lk, [this] { - return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); - }); - if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { - return; - } - req = std::move(*mtp_pending); - mtp_pending.reset(); - mtp_in_flight = true; - } - - mtp_response resp; - resp.status = decode_mtp_run(req, resp); - - { - std::lock_guard lk(mtp_mu); - mtp_in_flight = false; - mtp_completed = std::move(resp); - } - mtp_cv_response.notify_one(); - } -} - -// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV -// read retired), then sync sched_mtp — WITHOUT consuming mtp_completed (the spec loop's _wait -// still needs the drafts). Called at decode() top before any KV mutation. Inert when no -// assistant is loaded or nothing is in flight (drained==false ⇒ no extra sched sync). -void llama_context::mtp_drain_before_mutate() { - if (!model.mtp_assistant) { - return; - } - bool drained = false; - { - std::unique_lock lk(mtp_mu); - if (mtp_pending.has_value() || mtp_in_flight) { - mtp_cv_response.wait(lk, [this] { - return !mtp_in_flight && !mtp_pending.has_value(); - }); - drained = true; - } - } - if (drained && sched_mtp) { - ggml_backend_sched_synchronize(sched_mtp.get()); - } -} - llm_graph_params llama_context::graph_params( llm_graph_result * res, const llama_ubatch & ubatch, @@ -5177,24 +4381,6 @@ int32_t llama_decode( return ret; } -// opencoti F5 M6-S4 mtp (P2 inert) -int32_t llama_decode_mtp( - llama_context * ctx, - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last) { - if (!ctx) { - LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); - return -1; - } - return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); -} - int32_t llama_decode_mtp_fused_nextn( llama_context * ctx, llama_seq_id seq_id, diff --git a/llama.cpp/src/llama-context.h b/llama.cpp/src/llama-context.h index b6b0efb..32034fd 100644 --- a/llama.cpp/src/llama-context.h +++ b/llama.cpp/src/llama-context.h @@ -147,48 +147,8 @@ struct llama_context { llama_memory_context_i * mctx, ggml_status & ret); - // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. - llm_graph_params graph_params_mtp( - llm_graph_result * res, - const llama_ubatch & ubatch, - const llama_memory_context_i * mctx) const; - - int32_t decode_mtp( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last); - - int32_t decode_mtp_sync( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_logits, - float * out_h_prev_last); - - // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps - // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft - // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced - // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits - // verification path stays on decode_mtp_sync. - int32_t decode_mtp_fused( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - float * h_prev, - int32_t n_steps, - llama_token * out_drafts, - float * out_h_prev_last); - // opencoti fused-NextN (#590): fused N-step greedy draft on a DECODER_MTP (Qwen NextN) draft - // context using the unified cache. Twin of decode_mtp_fused minus mtp_assistant/sched_mtp/iSWA. + // context using the unified cache (the only in-context MTP draft driver). int32_t decode_mtp_fused_nextn( llama_seq_id seq_id, llama_pos attn_pos, @@ -198,25 +158,6 @@ struct llama_context { llama_token * out_drafts, float * out_h_prev_last); - // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync - // facade — out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams - // no per-step logits); otherwise it submits via decode_mtp_async and blocks in - // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the - // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). - int32_t decode_mtp_async( - llama_seq_id seq_id, - llama_pos attn_pos, - llama_token last_token, - const float * h_prev, - int32_t n_steps); - int32_t decode_mtp_wait( - llama_token * out_drafts, - float * out_h_prev_last); - // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before - // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP - // assistant is loaded or nothing is in flight (the common case). - void mtp_drain_before_mutate(); - int encode(const llama_batch & batch_inp); int decode(const llama_batch & batch_inp); @@ -333,14 +274,6 @@ private: const llama_memory_context_i * mctx, llm_graph_type gtype) const; - // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. - bool ensure_sched_mtp(); - llm_graph_result * process_ubatch_mtp( - const llama_ubatch & ubatch, - llama_memory_context_i * mctx, - ggml_status & ret); - ggml_status graph_compute_mtp(ggml_cgraph * gf); - llm_graph_cb graph_get_cb() const; // TODO: read/write lora adapters and cvec @@ -440,58 +373,22 @@ private: llm_graph_result_ptr gf_res_prev; llm_graph_result_ptr gf_res_reserve; - // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be - // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable - // single-token MTP graph result across steps of one decode_mtp call. - ggml_backend_sched_ptr sched_mtp; - llm_graph_result_ptr gf_res_prev_mtp; - // opencoti #590/bug-858: DEDICATED scheduler + result for the fused NextN self-spec draft // (decode_mtp_fused_nextn). It must NOT share the main sched/gf_res_prev with target decode: // every interleaved target decode RESETS the main sched (reallocating compute buffers) and // gf_res_prev, so a fused graph parked there can never satisfy can_reuse AND its tensor memory is // clobbered → 100% rebuild + CUDA-recapture per draft round (measured: build 0.5ms + comp-launch - // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched (like the Gemma assistant's - // sched_mtp, but ensure_sched_mtp is assistant-only: needs mtp_assistant + iswa cache) isolates the + // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched isolates the // fused graph so can_reuse compares fused-vs-fused (stable within a 256-cell n_kv bucket) and the // ggml graph + CUDA capture persist across rounds. NextN and Gemma-assistant are mutually exclusive. ggml_backend_sched_ptr sched_nextn; llm_graph_result_ptr gf_res_prev_nextn; int32_t nextn_reserved_steps = 0; - // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for - // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it - // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so - // ensure_sched_mtp only re-reserves when a larger fused block is requested. + // opencoti #590 fused NextN: number of draft steps to unroll into the MTP graph for the + // NEXT build (decode_mtp_fused_nextn sets it to n_steps around its build; the normal + // decode/reserve paths leave it at 1 so graph_params stays single-step). int32_t mtp_fused_steps = 1; - int32_t mtp_reserved_steps = 0; - - // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. - // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp - // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync - // facade), so the worker never races the target's sched. The destructor joins the worker. - struct mtp_request { - llama_seq_id seq_id = 0; - llama_pos attn_pos = 0; - llama_token last_token = 0; - std::vector h_prev; - int32_t n_steps = 0; - }; - struct mtp_response { - int32_t status = 0; - std::vector drafts; - std::vector h_prev_last; - }; - std::thread mtp_worker; - std::atomic mtp_worker_stop{false}; - std::mutex mtp_mu; - std::condition_variable mtp_cv_request; - std::condition_variable mtp_cv_response; - std::optional mtp_pending; // submitted, not yet picked up - bool mtp_in_flight = false; // worker is processing - std::optional mtp_completed; // worker finished, awaiting _wait - int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop - void mtp_worker_loop(); // host buffer for the model output (logits and embeddings) ggml_backend_buffer_ptr buf_output; diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp index e0d712e..6fc9679 100644 --- a/llama.cpp/src/llama-graph.cpp +++ b/llama.cpp/src/llama-graph.cpp @@ -124,7 +124,7 @@ void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] - // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. + // (= attn_pos + 1 + k, filled by decode_mtp_fused_nextn). Single-step path leaves this empty. // bug-870: mrope archs (qwen35) size each step's pos tensor I32[4] — fill the M-RoPE // text-token layout [p,p,p,0] (mirrors llm_graph_input_pos::set_input); standard rope = [p]. for (size_t k = 0; k < inp_pos_steps.size(); ++k) { @@ -570,7 +570,7 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently — // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP - // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but + // dual-ctx assistant draft graph shares the target KV through mem_other, but // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP @@ -2999,95 +2999,9 @@ ggml_tensor * llm_graph_context::build_attn( return cur; } -// opencoti F5 M6-S4 mtp -ggml_tensor * llm_graph_context::build_attn_mtp( - llm_graph_input_attn_kv_iswa * inp, - ggml_tensor * wo, - ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * kq_b, - ggml_tensor * sinks, - ggml_tensor * v_mla, - float kq_scale, - int il_mtp, - int32_t il_kv_tgt, - bool read_from_swa_kv, - int64_t kv_embd_head_v, - int64_t kv_n_head_v, - bool use_k_as_v) const { - auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; - auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; - - if (k_rot) { - q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); - } - if (v_rot) { - // V-only rotation is applied after MHA in the standard path; no V write here. - } - - ggml_build_forward_expand(gf, q_cur); - - const auto * mctx_iswa = inp->mctx; - const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); - - const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); - - ggml_tensor * q = q_cur; - ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); - ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); - - if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0 - || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) - // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the - // authoritative fused-turbo path — it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp - // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at - // head_dim∈{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 - // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired - // inverse on the output → wrong logits (graph still *built*, which is why P2's inert build - // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity - // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. - if (q->ne[0] % 128 != 0) { - const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; - q = ggml_pad(ctx0, q, pad, 0, 0, 0); - } - if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } - } - - ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); - cb(cur, "kqv_out_mtp", il_mtp); - - if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0 - || v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) - const int64_t orig_v_head = kv_embd_head_v; - const int64_t padded_v_head = v->ne[0]; - if (padded_v_head != orig_v_head) { - const int64_t n_head_v = kv_n_head_v; - const int64_t n_tokens_cur = cur->ne[1]; - cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); - cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, - cur->nb[1], cur->nb[2], 0); - cur = ggml_cont(ctx0, cur); - cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); - } - } - - if (v_rot) { - cur = ggml_mul_mat_aux(ctx0, cur, v_rot); - } - - if (wo) { - cur = build_lora_mm(wo, cur); - cb(cur, "mtp_wo_out", il_mtp); - } - if (wo_b) { - cur = ggml_add(ctx0, cur, wo_b); - } - - return cur; -} // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the frozen prefix -// KV on the UNIFIED cache. Twin of build_attn_mtp for the non-iSWA llm_graph_input_attn_kv used by +// KV on the UNIFIED cache via the non-iSWA llm_graph_input_attn_kv used by // Qwen NextN drafters. Reads mctx->get_k/get_v(il) with the input's kq_mask — which exposes the // full prefix 0..pmax because get_n_kv reports the cache's used extent, not the mtp_slot_info cell // count. Writes NO draft KV; recurrence is carried by the hidden-state chain. wo is applied by the diff --git a/llama.cpp/src/llama-graph.h b/llama.cpp/src/llama-graph.h index d3a3499..daa61b0 100644 --- a/llama.cpp/src/llama-graph.h +++ b/llama.cpp/src/llama-graph.h @@ -595,7 +595,7 @@ struct llm_graph_params { // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone // into the next step with no host round-trip / per-step GPU sync (the throughput fix). - // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. + // Read by the fused NextN builders; default 1 keeps every non-MTP construction unchanged. int32_t n_mtp_steps = 1; // return true if the "other" params would result in a graph with the same topology as with the current params @@ -1021,28 +1021,8 @@ struct llm_graph_context { int il, bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) - // opencoti F5 M6-S4 mtp - // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). - // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). - // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). - ggml_tensor * build_attn_mtp( - llm_graph_input_attn_kv_iswa * inp, - ggml_tensor * wo, - ggml_tensor * wo_b, - ggml_tensor * q_cur, - ggml_tensor * kq_b, - ggml_tensor * sinks, - ggml_tensor * v_mla, - float kq_scale, - int il_mtp, - int32_t il_kv_tgt, - bool read_from_swa_kv, - int64_t kv_embd_head_v, - int64_t kv_n_head_v, - bool use_k_as_v) const; - // opencoti fused-NextN (Option A): read-only cross-attention into the frozen prefix KV on the - // UNIFIED cache. Twin of build_attn_mtp for llm_graph_input_attn_kv (non-iSWA). Reads + // UNIFIED cache (llm_graph_input_attn_kv). Reads // mctx->get_k/get_v(il) with the input's kq_mask (which exposes 0..pmax via get_n_kv's used // extent), runs build_attn_mha, writes NO draft KV. wo is applied by the caller. // See docs/features/fused_nextn_mtp.md. diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp index 9628df1..f2e95a0 100644 --- a/llama.cpp/src/llama-kv-cache-iswa.cpp +++ b/llama.cpp/src/llama-kv-cache-iswa.cpp @@ -247,21 +247,6 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_full() { return std::make_unique(this); } -// opencoti F5 M6-S4 mtp -llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { - llama_kv_cache::slot_info_vec_t sinfos_base; - llama_kv_cache::slot_info_vec_t sinfos_swa; - - sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); - sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); - - std::vector ubatches; - ubatches.push_back(std::move(ubatch)); - - return std::make_unique( - this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); -} - llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { return std::make_unique(this, lctx, optimize); } diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h index 06fd1ae..a7eec3d 100644 --- a/llama.cpp/src/llama-kv-cache-iswa.h +++ b/llama.cpp/src/llama-kv-cache-iswa.h @@ -77,8 +77,6 @@ public: llama_memory_context_ptr init_full() override; - // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). - llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp index 4416d38..71bf0d5 100644 --- a/llama.cpp/src/llama-kv-cache.cpp +++ b/llama.cpp/src/llama-kv-cache.cpp @@ -5439,7 +5439,7 @@ llama_kv_cache_context::llama_kv_cache_context( llama_kv_cache::slot_info_vec_t sinfos, std::vector ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, - // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from + // which builds its graph inline and never calls apply()) get a valid mask/K/V shape from // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a // view with a garbage dim-2 (observed ne[2]=2956782432 → ggml.c view-bounds assert). For // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. diff --git a/llama.cpp/src/models/gemma4-assistant.cpp b/llama.cpp/src/models/gemma4-assistant.cpp index cab4fa8..64ea519 100644 --- a/llama.cpp/src/models/gemma4-assistant.cpp +++ b/llama.cpp/src/models/gemma4-assistant.cpp @@ -4,380 +4,10 @@ #include #include -static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { - p.arch = mtp_model.arch; - p.hparams = mtp_model.hparams; - p.gtype = LLM_GRAPH_TYPE_MTP; - return p; -} - -// Last layer in [range_start, range_end) whose attention type matches want_swa. -static int32_t gemma4_mtp_kv_layer_last_in_range( - const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { - int32_t best = -1; - if (range_start < 0) { - range_start = 0; - } - if (range_end > (int32_t) tgt.n_layer) { - range_end = (int32_t) tgt.n_layer; - } - for (int32_t il = range_start; il < range_end; ++il) { - if (tgt.is_swa((uint32_t) il) == want_swa) { - best = il; - } - } - return best; -} - -// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> -// (post-projected backbone hidden, vocabulary logits, argmax token id). -// -// Used by both the single-step path (n_mtp_steps==1) and the chained path -// (n_mtp_steps>1, called n_mtp_steps times within the same graph). -// -// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position -// for this step's RoPE (the cross-attn mask is shared across steps because the -// target's KV cache only contains positions ≤ attn_pos and all step positions -// are > attn_pos, so causal/SWA admit all target cells uniformly). -// -// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too — required for greedy match with verify). -// `out_argmax` (I32 [1]): greedy token id on-device. -static void gemma4_mtp_build_one_step( - const llm_graph_context & gctx, - const llama_model & target, - const llama_model & mtp, - llm_graph_input_attn_kv_iswa * inp_attn, - ggml_tensor * tok_step, // I32 [1] - ggml_tensor * h_step, // F32 [n_bb, 1] - ggml_tensor * pos_step, // I32 [1] - ggml_tensor ** out_logits, // F32 [n_vocab, 1] - ggml_tensor ** out_h_post, // F32 [n_bb, 1] - ggml_tensor ** out_argmax) { // I32 [1] - auto * ctx0 = gctx.ctx0; - auto * gf = gctx.gf; - const auto & hparams = gctx.hparams; - const auto & cparams = gctx.cparams; - const int n_layer = (int) gctx.n_layer; - const int n_tokens = (int) gctx.n_tokens; - const int n_ctx_orig = (int) gctx.n_ctx_orig; - const int rope_type = gctx.rope_type; - const float ext_factor = gctx.ext_factor; - const float attn_factor = gctx.attn_factor; - const float beta_fast = gctx.beta_fast; - const float beta_slow = gctx.beta_slow; - auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; - - const float tok_scale = sqrtf((float) target.hparams.n_embd); - - ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); - cb(tok_e, "mtp_tgt_tok_embd", -1); - - // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). - // Use target n_embd so Edge / non-Edge targets match the main forward. - tok_e = ggml_scale(ctx0, tok_e, tok_scale); - cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); - - ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); - cb(inp_cat, "mtp_concat", -1); - - ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); - cb(inpL, "mtp_pre_proj_out", -1); - - ggml_build_forward_expand(gf, inpL); - - ggml_tensor * cur = nullptr; - - for (int il = 0; il < n_layer; ++il) { - const int64_t n_embd_head = hparams.n_embd_head_k(il); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); - - const int64_t n_head = hparams.n_head(il); - - const float freq_base_l = mtp.get_rope_freq_base(cparams, il); - const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - - cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); - cb(cur, "attn_norm", il); - - ggml_tensor * freq_factors = nullptr; - if (!hparams.is_swa(il)) { - freq_factors = mtp.layers[il].rope_freqs; - } - - ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); - - Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); - cb(Qcur, "Qcur_normed", il); - - Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_pos", il); - - const bool read_swa = hparams.is_swa(il); - - const int32_t n_tgt = (int32_t) target.hparams.n_layer; - - // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads - // ONE shared KV per attention type from the target — the LAST layer of that type. - // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py - // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} - const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); - - GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); - - // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and - // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the - // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must - // ALWAYS fetch V from the cache — not reuse the post-RoPE K tensor. - const bool use_k_as_v = false; - - const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); - const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); - - cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, - hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); - - cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); - cb(cur, "attn_post_norm", il); - - ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); - cb(attn_out, "attn_out", il); - - GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); - - cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); - cb(cur, "ffn_norm", il); - - cur = gctx.build_ffn(cur, - mtp.layers[il].ffn_up, nullptr, nullptr, - mtp.layers[il].ffn_gate, nullptr, nullptr, - mtp.layers[il].ffn_down, nullptr, nullptr, - nullptr, - LLM_FFN_GELU, LLM_FFN_PAR, il); - cb(cur, "ffn_out", il); - - cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); - cb(cur, "ffn_post_norm", il); - - cur = ggml_add(ctx0, cur, attn_out); - - if (mtp.layers[il].out_scale) { - cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); - cb(cur, "out_scaled", il); - } - - cur = gctx.build_cvec(cur, il); - cb(cur, "l_out", il); - - inpL = cur; - } - - cur = inpL; - - cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); - cb(cur, "result_norm", -1); - - ggml_tensor * h_inner = cur; - - ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); - cb(backbone, "mtp_post_proj_out", -1); - - const int64_t n_vocab = mtp.tok_embd->ne[1]; - const int64_t n_tokens_mtp = h_inner->ne[1]; - - if (mtp.hparams.use_ordered_embeddings) { - // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full - // [n_vocab] row then argmax — matches masked full-vocab greedy (sparse-only argmax broke server accept). - GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); - GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); - const uint32_t n_c = mtp.hparams.n_centroids; - const uint32_t top_k = mtp.hparams.centroid_top_k; - GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); - GGML_ASSERT(n_vocab % (int64_t) n_c == 0); - const int64_t vsc = n_vocab / (int64_t) n_c; - - ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); - cb(centroid_logits, "mtp_centroid_logits", -1); - - ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); - cb(topk_idx, "mtp_centroid_topk_idx", -1); - - const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); - ggml_tensor * ordering = ggml_view_2d( - ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); - cb(ordering, "mtp_token_ordering_view", -1); - - ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); - cb(sel_ids, "mtp_selected_token_ids", -1); - - const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; - ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); - cb(flat_ids, "mtp_selected_token_ids_flat", -1); - - ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); - cb(sel_emb, "mtp_selected_embd", -1); - - ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); - cb(sel_logits, "mtp_selected_logits", -1); - ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); - - ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); - logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); - cb(logits_full, "mtp_logits_masked_base", -1); - - ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); - ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); - cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); - cb(cur, "mtp_logits_scatter_view", -1); - cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); - cb(cur, "mtp_logits_full", -1); - } else { - cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); - cb(cur, "result_output_dense", -1); - } - - if (hparams.f_final_logit_softcapping) { - cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); - cur = ggml_tanh(ctx0, cur); - cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); - } - - cb(cur, "result_output", -1); - - // Greedy argmax on-device: I32 [1] token index into the vocabulary row. - ggml_tensor * arg = ggml_argmax(ctx0, cur); - cb(arg, "result_argmax", -1); - - *out_logits = cur; - *out_h_post = backbone; - *out_argmax = arg; -} - -llm_build_gemma4_mtp::llm_build_gemma4_mtp( - const llama_model & target_model, - const llama_model & mtp_model, - const llm_graph_params & params) : - llm_graph_context(graph_params_for_mtp(params, mtp_model)), - target(target_model), - mtp(mtp_model) { - const int64_t n_bb = mtp.hparams.n_embd_out_impl; - GGML_ASSERT(n_bb > 0); - GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); - - // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 - // (fused path) chain from the previous step's on-device argmax + post-projection, so only - // step 0 reads token/h from the host. - ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); - ggml_set_input(inp_tok); - cb(inp_tok, "mtp_inp_last_token", -1); - - ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); - ggml_set_input(inp_h); - cb(inp_h, "mtp_inp_h_prev", -1); - - // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into - // ONE graph (no per-step host round-trip / GPU sync — the throughput fix). n_steps==1 is the - // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). - const int32_t n_steps = std::max(params.n_mtp_steps, 1); - - auto * inp_attn = build_attn_inp_kv_iswa(); - - if (n_steps <= 1) { - { - auto inp_wrap = std::make_unique(); - inp_wrap->inp_last_token = inp_tok; - inp_wrap->inp_h_prev = inp_h; - res->add_input(std::move(inp_wrap)); - } - - ggml_tensor * inp_pos = build_inp_pos(); - - ggml_tensor * logits = nullptr; - ggml_tensor * h_post = nullptr; - ggml_tensor * arg = nullptr; - gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, - inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); - - res->t_embd = h_post; - res->t_logits = logits; - res->t_argmax = arg; - - ggml_build_forward_expand(gf, arg); - ggml_build_forward_expand(gf, h_post); - return; - } - - // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is - // shared across steps — see gemma4_mtp_build_one_step's contract). The chain ties step k+1's - // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. - auto inp_wrap = std::make_unique(); - inp_wrap->inp_last_token = inp_tok; - inp_wrap->inp_h_prev = inp_h; - inp_wrap->inp_pos_steps.reserve(n_steps); - - std::vector pos_steps; - pos_steps.reserve(n_steps); - for (int32_t k = 0; k < n_steps; ++k) { - ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); - ggml_set_input(p); - cb(p, "mtp_inp_pos_step", k); - pos_steps.push_back(p); - inp_wrap->inp_pos_steps.push_back(p); - } - res->add_input(std::move(inp_wrap)); - - ggml_tensor * tok_k = inp_tok; - ggml_tensor * h_k = inp_h; - - std::vector step_args; - step_args.reserve(n_steps); - ggml_tensor * last_logits = nullptr; - ggml_tensor * last_h_post = nullptr; - - for (int32_t k = 0; k < n_steps; ++k) { - ggml_tensor * logits_k = nullptr; - ggml_tensor * h_post_k = nullptr; - ggml_tensor * arg_k = nullptr; - gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, - tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); - - step_args.push_back(arg_k); - last_logits = logits_k; - last_h_post = h_post_k; - - // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the - // target vocab — already used as a get_rows index inside build_one_step), next step's - // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. - tok_k = arg_k; - h_k = h_post_k; - } - - // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 - // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend — a - // negligible side-consumer that does NOT touch the on-device get_rows chain above. - ggml_tensor * all_args = step_args[0]; - for (int32_t k = 1; k < n_steps; ++k) { - all_args = ggml_concat(ctx0, all_args, step_args[k], 0); - } - cb(all_args, "mtp_fused_argmax", -1); - - res->t_argmax = all_args; // I32 [n_steps] — the drafted token block - res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed - res->t_logits = last_logits; // unused on the live path (out_logits == NULL) - - ggml_build_forward_expand(gf, all_args); - ggml_build_forward_expand(gf, last_h_post); -} - // opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via // llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path -// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the -// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 -// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. +// (llama_model_mapping -> this class -> load_arch_hparams/tensors). The assistant is never a +// primary model; build_arch_graph builds the dual-context drafter graph (ctx_other required). void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); @@ -443,7 +73,7 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { // // opencoti bug-858 (dual-context MTP): pre_projection is classified LLM_TENSOR_LAYER_INPUT, so the // plain create_tensor lands it on the CPU buft. The single-context facade tolerates that (it runs on - // the target's sched_mtp, which has the CPU backend registered), but a REAL draft context ctx_dft + // the target's scheduler, which has the CPU backend registered), but a REAL draft context ctx_dft // (llama_init_from_model over the nested assistant) registers only the assistant's *offloaded* (CUDA) // weight buffers — a CPU-resident weight then reads back uninitialized → the whole drafter forward is // NaN → 0% accept. Force it onto the output-head (GPU-when-offloaded) buft, exactly like tok_embd @@ -502,14 +132,12 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { // wk/wv): each attention layer reads the TARGET's shared K/V *in place* via the standard iSWA // build_attn (Qcur only, null k_cur/v_cur -> no store). The draft context's create_memory (A3) // shares the target's KV cells and aliases the target's last full (n_layer-1) / SWA (n_layer-2) -// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V — -// which is exactly what the single-context path did explicitly via build_attn_mtp(..., il_kv). +// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V. // The input token embedding + backbone hidden come from the target model via ctx_other. +// (The retired single-context facade did this explicitly via a build_attn_mtp cross-read; removed +// in #611 once this dual-context path proved parity — see UPSTREAM_SYNC.md.) // -// This is DISTINCT from the single-context llm_build_gemma4_mtp (built by the gemma4 TARGET's -// build_arch_graph, gemma4.cpp) which is kept fully intact as the fallback drafter. -// -// Faithful to upstream b9859 (and unlike our single-context build): dense LM head over tok_embd +// Faithful to upstream b9859: dense LM head over tok_embd // (no ordered-embeddings/centroid path — b9859 loads centroids but ignores them in the graph), // no final-logit softcapping (monotonic -> argmax-invariant), and no control-vector application // (inert without a control vector). See docs/evaluations/mtp.md. @@ -655,8 +283,6 @@ struct llm_build_gemma4_assistant_dual : public llm_graph_context { std::unique_ptr llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { // opencoti bug-858 dual-context MTP (A6): build the gemma4-assistant drafter graph. This model // is KV-less and only valid as an MTP draft context (ctx_dft) created FROM it with - // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. The single-context - // engine (llm_build_gemma4_mtp) is built by the gemma4 TARGET's build_arch_graph instead and - // never reaches here; it remains the fallback drafter. + // cparams.ctx_other = the Gemma 4 target — the ctor GGML_ASSERTs that. return std::make_unique(*this, params); } diff --git a/llama.cpp/src/models/gemma4.cpp b/llama.cpp/src/models/gemma4.cpp index bdf5990..464e5f5 100644 --- a/llama.cpp/src/models/gemma4.cpp +++ b/llama.cpp/src/models/gemma4.cpp @@ -132,14 +132,6 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { - // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), - // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF - // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ - // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. - if (params.gtype == LLM_GRAPH_TYPE_MTP) { - GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); - return std::make_unique(*this, *mtp_assistant, params); - } return std::make_unique(*this, params); } diff --git a/llama.cpp/src/models/models.h b/llama.cpp/src/models/models.h index deb2566..6efae4a 100644 --- a/llama.cpp/src/models/models.h +++ b/llama.cpp/src/models/models.h @@ -809,19 +809,10 @@ struct llama_model_gemma4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; -// opencoti F5 M6-S4 mtp -// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. -struct llm_build_gemma4_mtp : public llm_graph_context { - const llama_model & target; - const llama_model & mtp; - - llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); -}; - // opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant — loaded INTO a gemma4 target via // llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own -// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the -// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). +// hparams/tensors; build_arch_graph builds the dual-context drafter graph (requires +// cparams.ctx_other = the Gemma 4 target, bug-858). struct llama_model_gemma4_assistant : public llama_model_base { llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/llama.cpp/src/models/qwen35.cpp b/llama.cpp/src/models/qwen35.cpp index 3a21d69..827779e 100644 --- a/llama.cpp/src/models/qwen35.cpp +++ b/llama.cpp/src/models/qwen35.cpp @@ -628,7 +628,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. - // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. + // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); } else { Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, @@ -700,8 +700,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr return; } - // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant - // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One + // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver + // sets n_mtp_steps>1 (S3). One // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, diff --git a/llama.cpp/src/models/qwen35moe.cpp b/llama.cpp/src/models/qwen35moe.cpp index 0eb2871..af9ea1e 100644 --- a/llama.cpp/src/models/qwen35moe.cpp +++ b/llama.cpp/src/models/qwen35moe.cpp @@ -680,7 +680,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm // context's prior decodes; we do NOT write draft KV here — recurrence is carried by the // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here → pruned. - // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. + // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); } else { Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, @@ -784,8 +784,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm return; } - // opencoti-hook: fused-nextn-mtp — fused N-step draft chain (mirrors gemma4-assistant - // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One + // opencoti-hook: fused-nextn-mtp — fused N-step draft chain. DORMANT until the driver + // sets n_mtp_steps>1 (S3). One // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 // ids/token — ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp index 34b50b1..04207b2 100644 --- a/llama.cpp/tools/server/server-context.cpp +++ b/llama.cpp/tools/server/server-context.cpp @@ -918,25 +918,21 @@ private: add_bos_token = llama_vocab_get_add_bos(vocab); - // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so - // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file — - // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at - // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from - // both native branches (separate-model draft below; NextN MTP-ctx-against-target further - // down). Detect it via the enabled types vector, load into the target, and leave - // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's - // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). + // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4-assistant draft head is a SEPARATE GGUF (so + // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file, + // then runs as a REAL draft context (ctx_dft) whose cparams.ctx_other = ctx_tgt shares the + // target's KV (bug-858 dual-context, upstream b9859 shape). This is distinct from both + // native branches (separate-model draft below; NextN MTP-ctx-against-target further down). + // Detect it via the enabled types vector and load into the target first. const bool spec_assistant = std::find(params_base.speculative.types.begin(), params_base.speculative.types.end(), COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); if (params_base.speculative.has_dft() && spec_assistant) { - // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target - // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the - // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout - // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. - // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 - // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it - // explicitly and fail clean at boot instead of crashing mid-request. + // opencoti F5 M6-S4 mtp (#485/#611): with --parallel >1 WITHOUT --kv-unified the KV is + // per-stream-split (patch 0031); the assistant's shared-KV aliasing (mem_other) assumes the + // unified per-layer tensor layout, so per-stream splits are not supported. The unified + // cache IS handled (proven on the facade at --parallel 2 --kv-unified — the PolyKV config, + // which forces --kv-unified). Require it explicitly and fail clean at boot. if (params_base.n_parallel > 1 && !params_base.kv_unified) { SRV_ERR("%s", "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " @@ -965,21 +961,19 @@ private: return false; } - // opencoti bug-858 dual-context MTP (A5): optionally create a REAL draft context FROM the - // nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 + // opencoti bug-858 dual-context MTP (A5, default since #611): create a REAL draft context + // FROM the nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 // dual-context). The assistant is KV-less: its create_memory (A3) shares the target's KV // cells + aliases the last full/SWA layer, and its graph (A6) reads the target's tok_embd // + shared K/V via ctx_other; the shared-KV draft_mtp driver (is_mem_shared) then runs it. - // Gated by OPENCOTI_MTP_DUAL_CTX so the default keeps the proven single-context in-target - // path (ctx_dft == nullptr, llama_decode_mtp) byte-identical. - const char * mtp_dual_env = getenv("OPENCOTI_MTP_DUAL_CTX"); - const bool mtp_dual_ctx = mtp_dual_env && mtp_dual_env[0] && mtp_dual_env[0] != '0'; - if (mtp_dual_ctx) { + // This is the ONLY assistant execution path — the single-context in-target facade + // (llama_decode_mtp) was removed in #611 after the port proved accept parity. + { // llama_init_from_model wants a mutable model*; the nested assistant is owned mutably // by the target (unique_ptr), so the const_cast is safe. llama_model * assistant = const_cast(llama_model_get_mtp_assistant(model_tgt)); if (assistant == nullptr) { - SRV_ERR("%s", "MTP dual-context requested but the assistant is not loaded into the target\n"); + SRV_ERR("%s", "MTP assistant is not loaded into the target\n"); return false; } @@ -1002,11 +996,6 @@ private: params_base.speculative.draft.ctx_tgt = ctx_tgt; params_base.speculative.draft.ctx_dft = ctx_dft.get(); SRV_INF("%s", "MTP assistant dual-context draft created (ctx_other=target, shared KV)\n"); - } else { - // No separate draft context: the assistant decodes on ctx_tgt via llama_decode_mtp. - params_base.speculative.draft.ctx_tgt = ctx_tgt; - params_base.speculative.draft.ctx_dft = nullptr; - SRV_INF("%s", "MTP assistant loaded into target (single-context)\n"); } } else if (params_base.speculative.has_dft()) { // TODO speculative: move to common/speculative.cpp?