| From: opencoti |
| Subject: [PATCH 0096] remove single-context Gemma MTP facade; dual-ctx is the only assistant path (#611) |
|
|
| END of the bug-858 port: delete the retired single-context in-target assistant |
| engine (~1600 lines) now that the dual-context port (0093) proved accept parity: |
| - llama-context: decode_mtp facade + decode_mtp_sync/fused, async worker |
| (decode_mtp_async/wait/run, mtp_worker thread, drain guard), process_ubatch_mtp, |
| graph_params_mtp, ensure_sched_mtp, sched_mtp/gf_res_prev_mtp, graph_compute_mtp. |
| - public API llama_decode_mtp removed (llama_decode_mtp_fused_nextn stays). |
| - llama-graph: build_attn_mtp (iswa cross-read); kv-cache-iswa::init_mtp. |
| - models: llm_build_gemma4_mtp + one-step builder + gemma4.cpp MTP dispatch; |
| gemma4-assistant keeps ONLY the dual-ctx graph (ctx_other required). |
| - speculative: common_speculative_impl_draft_assistant + NDJSON tracer removed; |
| --spec-type draft-assistant now requires ctx_dft (server always creates it). |
| - server: OPENCOTI_MTP_DUAL_CTX env gate removed β dual-ctx unconditional; |
| the --parallel>1 -> --kv-unified boot guard is preserved (shared-KV aliasing |
| has the same per-stream-split constraint the facade had). |
| NextN side untouched (mtp_fused_steps, sched_nextn, mtp_slot_info, unified twin |
| builders); mtp_assistant model loading kept. Gate on bs2: A4B n2 305.4 tps / |
| 0.849 accept (== 0093 reference, boot line proves dual-ctx default), q35 n3 |
| 248.8 / 0.662. bug-2110 (par2 accept 0.635 on the tps harness) was RESOLVED as |
| a harness artifact: on the real 9-prompt mtp-bench the dual-ctx assistant holds |
| accept at --parallel 2 --kv-unified (ours 0.785 par1 / 0.783 par2; upstream |
| b9859 0.793 / 0.792). |
|
|
| |
| |
| |
| |
| @@ -305,7 +305,7 @@ struct common_params_speculative_draft { |
| |
| // opencoti F5 M6-S4 mtp β MTP (Gemma 4 assistant): OPTIONAL explicit draft-depth ceiling. |
| // 0 (default) = draft depth follows --spec-draft-n-max (n_max); the MTP head re-runs |
| - // autoregressively per step (decode_mtp_sync/fused), so depth is NOT limited to a fixed |
| + // autoregressively per step (draft_mtp / decode_mtp_fused_nextn), so depth is NOT limited to a fixed |
| // block size. bug-858: the old default 3 hard-capped ours at 2 drafts/round and blocked |
| // n-max scaling vs upstream b9859. When set >1, caps draft depth at draft_block_size-1. |
| int32_t draft_block_size = 0; |
| |
| |
| |
| |
| @@ -1409,315 +1409,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { |
| } |
| }; |
| |
| -// opencoti F5 M6-S4 mtp (P3 Phase B, restored onto 0.10.3): speculative draft head for Gemma-4 A4B. |
| -// The draft head (gemma4_assistant) lives INSIDE the target (loaded via llama_model_load_mtp_from_file, |
| -// P1); draft() reads the target's last backbone hidden state and cross-reads the target KV |
| -// (build_attn_mtp, P2) through the synchronous llama_decode_mtp facade. It emits all B-1 draft tokens |
| -// of a block in ONE facade call (no per-step sampler loop). |
| -// |
| -// This is a SIBLING of the upstream-native common_speculative_impl_draft_mtp (NextN self-attention), |
| -// not a replacement. Distinct enum values (_MTP = this assistant, _DRAFT_MTP = native NextN) never |
| -// collide, and both reuse the SAME seq-aware framework: per-seq hidden state is harvested in process() |
| -// and rolled forward in accept(), so the assistant drafter composes with PolyKV (--parallel >=2) |
| -// without any per-seq server plumbing. Hidden state is POST-norm output embeddings (need_embd). |
| - |
| -// Optional NDJSON tracer for MTP draft/accept events, gated by env LLAMA_MTP_ACC_TRACE. |
| -// unset/"0"/"" -> disabled, "1" -> stderr, else -> file path (append). One JSON object per line. |
| -namespace { |
| -struct mtp_acc_tracer { |
| - bool enabled = false; |
| - FILE * fp = nullptr; |
| - std::mutex mu; |
| - |
| - mtp_acc_tracer() { |
| - const char * v = std::getenv("LLAMA_MTP_ACC_TRACE"); |
| - if (!v || v[0] == '\0' || std::strcmp(v, "0") == 0) { |
| - return; |
| - } |
| - fp = (std::strcmp(v, "1") == 0) ? stderr : std::fopen(v, "a"); |
| - enabled = (fp != nullptr); |
| - } |
| - |
| - ~mtp_acc_tracer() { |
| - if (fp && fp != stderr) { |
| - std::fclose(fp); |
| - } |
| - } |
| - |
| - void writeln(const std::string & line) { |
| - if (!enabled) { |
| - return; |
| - } |
| - std::lock_guard<std::mutex> lk(mu); |
| - std::fputs(line.c_str(), fp); |
| - std::fputc('\n', fp); |
| - std::fflush(fp); |
| - } |
| -}; |
| - |
| -mtp_acc_tracer & mtp_tracer() { |
| - static mtp_acc_tracer t; |
| - return t; |
| -} |
| -} // namespace |
| - |
| -struct common_speculative_impl_draft_assistant : public common_speculative_impl { |
| - common_params_speculative_draft params; // reuses the draft params slot (ctx_tgt + draft_block_size) |
| - |
| - llama_context * ctx_tgt = nullptr; |
| - |
| - int32_t n_embd_backbone = 0; // h_prev dimension the MTP facade consumes |
| - int32_t n_embd_out = 0; // width of an llama_get_embeddings_ith row |
| - |
| - // Per-seq cross-batch hidden-state carry (POST-norm). process() harvests the target verify rows |
| - // for each seq into verify_h; accept() selects the last-accepted row into pending_h; draft() feeds |
| - // pending_h to llama_decode_mtp. Mirrors common_speculative_impl_draft_mtp's pending_h/verify_h |
| - // scheme so multi-seq (PolyKV) works without per-seq server plumbing. |
| - std::vector<std::vector<float>> pending_h; // [n_seq][n_embd_backbone] |
| - std::vector<std::vector<float>> verify_h; // [n_seq][n_rows * n_embd_backbone] |
| - std::vector<int32_t> verify_h_rows; |
| - std::vector<int32_t> i_batch_beg; |
| - std::vector<int32_t> i_batch_end; |
| - |
| - // Adaptive skip after consecutive zero-accept draft batches (per seq). When the MTP head |
| - // consistently mispredicts, drafting costs ~10ms with no accepted tokens; fall back to plain |
| - // verify-only for one batch. 0 = disabled; LLAMA_MTP_SKIP_STREAK_THRESHOLD = 1..32 to enable. |
| - int skip_streak_threshold = 0; |
| - std::vector<int> zero_accept_streak; |
| - std::vector<uint8_t> skip_last_draft; |
| - std::vector<size_t> prev_n_acc_at_draft; |
| - |
| - int trace_iter = 0; // tracing only (LLAMA_MTP_ACC_TRACE), no behavior change |
| - |
| - static float compute_h_l2(const float * h, int32_t n) { |
| - if (!h || n <= 0) { |
| - return 0.0f; |
| - } |
| - double s = 0.0; |
| - for (int32_t i = 0; i < n; ++i) { |
| - s += (double) h[i] * (double) h[i]; |
| - } |
| - return (float) std::sqrt(s); |
| - } |
| - |
| - common_speculative_impl_draft_assistant(const common_params_speculative & params, uint32_t n_seq) |
| - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_MTP, n_seq) |
| - , params(params.draft) |
| - { |
| - ctx_tgt = this->params.ctx_tgt; |
| - GGML_ASSERT(ctx_tgt && "MTP assistant requires ctx_tgt to be set"); |
| - |
| - const llama_model * model_tgt = llama_get_model(ctx_tgt); |
| - n_embd_backbone = (int32_t) llama_model_mtp_n_embd_backbone(model_tgt); |
| - n_embd_out = llama_model_n_embd_out(model_tgt); |
| - |
| - LOG_INF("%s: adding speculative implementation 'draft-assistant' (Gemma-4 MTP)\n", __func__); |
| - LOG_INF("%s: - n_max=%d, draft_block_size=%d, n_embd_backbone=%d, n_embd_out=%d\n", __func__, |
| - this->params.n_max, this->params.draft_block_size, n_embd_backbone, n_embd_out); |
| - |
| - // MTP reads the target's last backbone hidden state; keep embeddings on across decodes. |
| - llama_set_embeddings(ctx_tgt, true); |
| - |
| - if (const char * e = std::getenv("LLAMA_MTP_SKIP_STREAK_THRESHOLD")) { |
| - const int v = std::atoi(e); |
| - if (v >= 1 && v <= 32) { |
| - skip_streak_threshold = v; |
| - } |
| - } |
| - |
| - pending_h.assign(n_seq, std::vector<float>((size_t) std::max(1, n_embd_backbone), 0.0f)); |
| - verify_h.assign(n_seq, {}); |
| - verify_h_rows.assign(n_seq, 0); |
| - i_batch_beg.assign(n_seq, -1); |
| - i_batch_end.assign(n_seq, -1); |
| - zero_accept_streak.assign(n_seq, 0); |
| - skip_last_draft.assign(n_seq, 0); |
| - prev_n_acc_at_draft.assign(n_seq, 0); |
| - } |
| - |
| - ~common_speculative_impl_draft_assistant() override = default; |
| - |
| - void begin(llama_seq_id seq_id, const llama_tokens & /*prompt*/) override { |
| - llama_set_embeddings(ctx_tgt, true); |
| - if (seq_id >= 0 && seq_id < (llama_seq_id) n_seq) { |
| - skip_last_draft[seq_id] = 0; |
| - zero_accept_streak[seq_id] = 0; |
| - } |
| - } |
| - |
| - // Harvest the target's POST-norm hidden rows for this verify batch, grouped by seq, so accept() |
| - // can select the last-accepted token's hidden state. The spec framework calls process() after the |
| - // target decode, so embeddings are populated. No decode here (unlike native draft_mtp, which |
| - // mirrors into a separate ctx_dft) β the assistant reads the target's own embeddings directly. |
| - // NOTE (bug-858): PRE-norm harvest was TESTED and REFUTED β it collapses accept to ~0.05 at ALL |
| - // positions (incl short ctx). This drafter expects the post-norm hidden; do NOT swap to pre_norm. |
| - bool process(const llama_batch & batch_in) override { |
| - if (batch_in.n_tokens <= 0 || batch_in.token == nullptr || batch_in.embd != nullptr) { |
| - return true; |
| - } |
| - if (n_embd_backbone <= 0) { |
| - return true; |
| - } |
| - |
| - const int32_t n_tokens = batch_in.n_tokens; |
| - std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); |
| - std::fill(i_batch_end.begin(), i_batch_end.end(), -1); |
| - |
| - for (int k = 0; k < n_tokens; ++k) { |
| - const llama_seq_id s = batch_in.seq_id[k][0]; |
| - if (s < 0 || s >= (llama_seq_id) n_seq) { |
| - continue; |
| - } |
| - i_batch_end[s] = k; |
| - if (i_batch_beg[s] < 0) { |
| - i_batch_beg[s] = k; |
| - } |
| - } |
| - |
| - const int32_t n_copy = std::min(n_embd_backbone, n_embd_out); |
| - const size_t row_bb = (size_t) n_embd_backbone * sizeof(float); |
| - |
| - for (llama_seq_id s = 0; s < (llama_seq_id) n_seq; ++s) { |
| - if (i_batch_beg[s] < 0) { |
| - continue; |
| - } |
| - const int32_t n_rows = i_batch_end[s] - i_batch_beg[s] + 1; |
| - verify_h_rows[s] = n_rows; |
| - verify_h[s].assign((size_t) n_rows * n_embd_backbone, 0.0f); |
| - |
| - for (int32_t i = 0; i < n_rows; ++i) { |
| - const float * h = llama_get_embeddings_ith(ctx_tgt, i_batch_beg[s] + i); |
| - if (h) { |
| - std::memcpy(verify_h[s].data() + (size_t) i * n_embd_backbone, h, |
| - (size_t) n_copy * sizeof(float)); |
| - } |
| - } |
| - // default pending_h to the last row (correct when all drafts accepted / on prefill) |
| - std::memcpy(pending_h[s].data(), |
| - verify_h[s].data() + (size_t) (n_rows - 1) * n_embd_backbone, row_bb); |
| - } |
| - |
| - return true; |
| - } |
| - |
| - void draft(common_speculative_draft_params_vec & dparams) override { |
| - if (n_embd_backbone <= 0) { |
| - return; |
| - } |
| - |
| - // bug-858: draft depth follows --spec-draft-n-max (n_max) and the room left in context |
| - // (dp.n_max = n_draft_max), NOT draft_block_size. The MTP head re-runs autoregressively per |
| - // step (decode_mtp_sync/fused build a fresh single-step graph each step: argmax -> next |
| - // last_token, h_post -> next h_prev), so depth is bounded by n_max, not a fixed block. The |
| - // legacy draft_block_size-1 ceiling (default 3 -> 2) hard-capped ours at 2 drafts/round and |
| - // blocked n-max scaling vs upstream b9859 (which drafts to n_max autoregressively) -> REMOVED. |
| - |
| - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { |
| - auto & dp = dparams[seq_id]; |
| - if (!dp.drafting) { |
| - continue; |
| - } |
| - |
| - // zero-accept skip-streak: if n_acc_drafts has not moved since this seq's previous draft, |
| - // the previous batch produced 0 accepted drafts. |
| - if (skip_last_draft[seq_id]) { |
| - skip_last_draft[seq_id] = 0; |
| - } else if (n_acc_drafts == prev_n_acc_at_draft[seq_id]) { |
| - zero_accept_streak[seq_id]++; |
| - } else { |
| - zero_accept_streak[seq_id] = 0; |
| - } |
| - if (skip_streak_threshold > 0 && zero_accept_streak[seq_id] >= skip_streak_threshold) { |
| - zero_accept_streak[seq_id] = 0; |
| - skip_last_draft[seq_id] = 1; |
| - prev_n_acc_at_draft[seq_id] = n_acc_drafts; |
| - continue; // empty result -> server falls back to single-token verify |
| - } |
| - |
| - int32_t n_steps = params.n_max > 0 ? params.n_max : 1; |
| - if (dp.n_max > 0) { |
| - n_steps = std::min(n_steps, dp.n_max); |
| - } |
| - if (n_steps <= 0) { |
| - prev_n_acc_at_draft[seq_id] = n_acc_drafts; |
| - continue; |
| - } |
| - |
| - float * h_prev = pending_h[seq_id].data(); |
| - |
| - llama_memory_t mem = llama_get_memory(ctx_tgt); |
| - llama_pos attn_pos = mem ? llama_memory_seq_pos_max(mem, seq_id) : (llama_pos) 0; |
| - if (attn_pos < 0) { |
| - attn_pos = 0; |
| - } |
| - |
| - std::vector<llama_token> out((size_t) n_steps, 0); |
| - const int32_t rc = llama_decode_mtp( |
| - ctx_tgt, seq_id, attn_pos, dp.id_last, h_prev, n_steps, |
| - /*out_drafts =*/ out.data(), |
| - /*out_logits =*/ nullptr, |
| - /*out_h_prev_last=*/ nullptr); |
| - |
| - prev_n_acc_at_draft[seq_id] = n_acc_drafts; |
| - |
| - if (rc != 0) { |
| - LOG_ERR("%s: llama_decode_mtp failed (%d) seq_id=%d\n", __func__, (int) rc, (int) seq_id); |
| - continue; |
| - } |
| - |
| - auto & result = *dp.result; |
| - for (int32_t i = 0; i < n_steps; ++i) { |
| - result.push_back(out[i]); |
| - } |
| - |
| - if (mtp_tracer().enabled) { |
| - std::ostringstream oss; |
| - oss << "{\"evt\":\"mtp_draft\",\"iter\":" << trace_iter |
| - << ",\"seq_id\":" << (int) seq_id |
| - << ",\"id_last\":" << (int) dp.id_last |
| - << ",\"attn_pos\":" << (int) attn_pos |
| - << ",\"n_steps\":" << n_steps |
| - << ",\"h_l2\":" << std::fixed << std::setprecision(4) |
| - << compute_h_l2(h_prev, n_embd_backbone) << ",\"drafts\":["; |
| - for (int32_t i = 0; i < n_steps; ++i) { |
| - if (i) oss << ','; |
| - oss << (int) out[i]; |
| - } |
| - oss << "]}"; |
| - mtp_tracer().writeln(oss.str()); |
| - } |
| - } |
| - } |
| - |
| - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { |
| - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { |
| - return; |
| - } |
| - const int32_t n_rows = verify_h_rows[seq_id]; |
| - if (n_rows <= 0 || n_embd_backbone <= 0) { |
| - return; |
| - } |
| - // Row 0 is the sampled token; row k is the k-th accepted draft. Point h_prev at the |
| - // last-accepted token's hidden state (clamped to the harvested rows). |
| - const int32_t i_h = std::min<int32_t>(n_accepted, n_rows - 1); |
| - std::memcpy(pending_h[seq_id].data(), |
| - verify_h[seq_id].data() + (size_t) i_h * n_embd_backbone, |
| - (size_t) n_embd_backbone * sizeof(float)); |
| - |
| - if (mtp_tracer().enabled) { |
| - std::ostringstream oss; |
| - oss << "{\"evt\":\"mtp_accept\",\"iter\":" << trace_iter |
| - << ",\"seq_id\":" << (int) seq_id |
| - << ",\"n_accepted\":" << (int) n_accepted << "}"; |
| - mtp_tracer().writeln(oss.str()); |
| - ++trace_iter; |
| - } |
| - } |
| - |
| - bool need_embd() const override { |
| - return true; // MTP reads POST-norm output embeddings (h_prev) |
| - } |
| -}; |
| |
| struct common_speculative { |
| common_speculative_draft_params_vec dparams; |
| @@ -1934,15 +1625,15 @@ common_speculative * common_speculative_init(common_params_speculative & params, |
| break; |
| } |
| case COMMON_SPECULATIVE_TYPE_MTP: { // opencoti F5 M6-S4 mtp: Gemma-4 assistant drafter |
| - // opencoti bug-858 dual-context MTP: when the server created a real ctx_dft FROM the |
| - // assistant model (ctx_other=ctx_tgt; --spec-type draft-assistant + OPENCOTI_MTP_DUAL_CTX), |
| - // run the shared-KV draft_mtp driver (upstream b9859 is_mem_shared). Otherwise fall |
| - // back to the proven single-context in-target assistant (ctx_dft == nullptr). |
| - if (config.params.draft.ctx_dft != nullptr) { |
| - impls.push_back(std::make_unique<common_speculative_impl_draft_mtp>(config.params, n_seq)); |
| - } else { |
| - impls.push_back(std::make_unique<common_speculative_impl_draft_assistant>(config.params, n_seq)); |
| + // opencoti bug-858 dual-context MTP (#611): the assistant ALWAYS runs as a real |
| + // ctx_dft created FROM the assistant model (ctx_other=ctx_tgt) through the shared-KV |
| + // draft_mtp driver (upstream b9859 is_mem_shared). The single-context in-target |
| + // facade was removed in #611 after the dual-context port proved accept parity. |
| + if (config.params.draft.ctx_dft == nullptr) { |
| + LOG_ERR("%s: draft-assistant requires the dual-context draft (server creates ctx_dft with ctx_other=target)\n", __func__); |
| + return nullptr; |
| } |
| + impls.push_back(std::make_unique<common_speculative_impl_draft_mtp>(config.params, n_seq)); |
| break; |
| } |
| case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { |
| @@ -2010,8 +1701,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, |
| // opencoti F5 M6-S4 mtp: the 0.10.1-era host-driven MTP helpers (is_mtmd_safe / all_impls_mtmd_safe / |
| // set_seq_id / set_h_idx) are intentionally NOT restored onto 0.10.3. The new seq-aware speculative |
| // framework drives the assistant drafter entirely through the per-seq process()/draft(dparams)/ |
| -// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete (see the restore note on |
| -// common_speculative_impl_draft_assistant above). |
| +// accept(seq_id,n) hooks, so the old global seq_id/h_idx plumbing is obsolete. |
| |
| void common_speculative_free(common_speculative * spec) { |
| if (spec == nullptr) { |
| |
| |
| |
| |
| @@ -1041,22 +1041,6 @@ extern "C" { |
| struct llama_context * ctx, |
| struct llama_batch batch); |
| |
| - // opencoti F5 M6-S4 mtp |
| - // Gemma 4 MTP greedy draft: from (last_token, h_prev backbone hidden) at attn_pos, |
| - // emit n_steps draft tokens into out_drafts (cross-reads the target KV for seq_id). |
| - // out_logits (optional, [n_vocab*n_steps]) and out_h_prev_last (optional, [n_bb]) |
| - // receive per-step logits and the final backbone hidden. Returns 0 on success. |
| - LLAMA_API int32_t llama_decode_mtp( |
| - struct llama_context * ctx, |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last); |
| - |
| // opencoti fused-NextN (#590): fuse N per-step NextN draft decodes into one graph on a |
| // DECODER_MTP (Qwen NextN) draft context. GREEDY-ONLY (no p_min early-stop). Returns 0 on |
| // success. last_token must already be resident in ctx's cache at attn_pos (seed decode first). |
| |
| |
| |
| |
| @@ -570,8 +570,8 @@ enum llm_tensor { |
| |
| // opencoti F5 M6-S4 mtp β Gemma 4 assistant-MTP tensors, names upstream-aligned to #23398/#24282 |
| // (GGUF prefixes nextn.* / masked_embd_*). Data-plane identity == upstream so a future llamafile |
| - // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine stays our |
| - // single-context build_attn_mtp cross-read β NOT upstream's ctx_other. See plan. |
| + // bump to a base carrying #23398 reconciles these as a clean delete; the runtime engine is the |
| + // bug-858 dual-context port (cparams.ctx_other, upstream-shaped). See plan. |
| LLM_TENSOR_NEXTN_PROJ_PRE, |
| LLM_TENSOR_NEXTN_PROJ_POST, |
| LLM_TENSOR_MASKED_EMBD_CENTROIDS, |
| |
| |
| |
| |
| @@ -9,7 +9,7 @@ |
| #include "llama-memory.h" |
| #include "llama-mmap.h" |
| #include "llama-model.h" |
| -#include "llama-kv-cache-iswa.h" // opencoti F5 M6-S4 mtp: dynamic_cast + init_mtp |
| +#include "llama-kv-cache-iswa.h" |
| #include "llama-ext.h" |
| #include "dca.h" // opencoti F5 dca (#618): dca_resolve_chunk_size for the chunk % n_ubatch == 0 guard |
| #include "llama.h" |
| @@ -527,17 +527,6 @@ llama_context::llama_context( |
| } |
| |
| llama_context::~llama_context() { |
| - // opencoti F5 M6-S4 mtp (P4): stop + join the async MTP draft worker before tearing down |
| - // sched_mtp and the backends it touches. |
| - if (mtp_worker.joinable()) { |
| - { |
| - std::lock_guard<std::mutex> lk(mtp_mu); |
| - mtp_worker_stop.store(true, std::memory_order_release); |
| - } |
| - mtp_cv_request.notify_all(); |
| - mtp_worker.join(); |
| - } |
| - |
| if (!model.hparams.no_alloc) { |
| for (size_t i = 0; i < backend_ptrs.size(); ++i) { |
| ggml_backend_t backend = backend_ptrs[i]; |
| @@ -1421,208 +1410,6 @@ bool llama_context::set_adapter_cvec( |
| return res; |
| } |
| |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -bool llama_context::ensure_sched_mtp() { |
| - // opencoti F5 M6-S4 mtp (P4 fused): the reserve must cover the requested fused-step count |
| - // (mtp_fused_steps). A fused N-step graph has ~NΓ the nodes of the single-step graph and a |
| - // distinct topology, so when a larger block is requested we drop and rebuild the reserve. |
| - const int32_t need = std::max(mtp_fused_steps, 1); |
| - if (sched_mtp && mtp_reserved_steps >= need) { |
| - return true; |
| - } |
| - if (!model.mtp_assistant) { |
| - return false; |
| - } |
| - sched_mtp.reset(); |
| - gf_res_prev_mtp.reset(); |
| - |
| - const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); |
| - const size_t max_nodes = (size_t) this->graph_max_nodes(n_tokens) * (size_t) need; |
| - |
| - gf_res_prev_mtp.reset(new llm_graph_result(max_nodes)); |
| - sched_mtp.reset(ggml_backend_sched_new( |
| - backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), |
| - max_nodes, /*pipeline_parallel*/ false, cparams.op_offload)); |
| - if (!sched_mtp) { |
| - LLAMA_LOG_ERROR("%s: ggml_backend_sched_new failed for sched_mtp\n", __func__); |
| - gf_res_prev_mtp.reset(); |
| - return false; |
| - } |
| - |
| - // Reserve a single-token MTP graph so backends allocate compute buffers on sched_mtp. |
| - // The MTP graph is invariant in size (always n_tokens=1, n_seqs=1, n_outputs=1), |
| - // so a single reserve covers all subsequent decode_mtp calls. |
| - { |
| - auto * kv_iswa = dynamic_cast<llama_kv_cache_iswa *>(memory.get()); |
| - if (!kv_iswa) { |
| - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory\n", __func__); |
| - sched_mtp.reset(); |
| - gf_res_prev_mtp.reset(); |
| - return false; |
| - } |
| - |
| - llama_memory_context_ptr mctx = memory->init_full(); |
| - if (!mctx) { |
| - LLAMA_LOG_ERROR("%s: failed to init memory context for MTP reserve\n", __func__); |
| - sched_mtp.reset(); |
| - gf_res_prev_mtp.reset(); |
| - return false; |
| - } |
| - |
| - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; |
| - auto data = std::make_shared<llama_ubatch::data_t>(); |
| - data->token.resize(1); |
| - data->embd.resize(n_bb); |
| - data->pos.resize(1); |
| - data->n_seq_id.resize(1); |
| - data->seq_id.resize(1); |
| - data->seq_id_data.resize(1); |
| - data->output.resize(1); |
| - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); |
| - data->seq_id_unq.push_back(0); |
| - data->seq_idx[0] = 0; |
| - data->n_seq_id[0] = 1; |
| - data->seq_id_data[0] = 0; |
| - data->seq_id[0] = &data->seq_id_data[0]; |
| - data->output[0] = 1; |
| - |
| - llama_ubatch ub{}; |
| - ub.b_equal_seqs = 1; |
| - ub.n_tokens = 1; |
| - ub.n_seq_tokens = 1; |
| - ub.n_seqs = 1; |
| - ub.n_seqs_unq = 1; |
| - ub.n_pos = 1; |
| - ub.token = data->token.data(); |
| - ub.embd = data->embd.data(); |
| - ub.pos = data->pos.data(); |
| - ub.n_seq_id = data->n_seq_id.data(); |
| - ub.seq_id = data->seq_id.data(); |
| - ub.seq_id_unq = data->seq_id_unq.data(); |
| - ub.seq_idx = data->seq_idx.data(); |
| - ub.output = data->output.data(); |
| - ub.data = data; |
| - |
| - const uint32_t save_n_outputs = n_outputs; |
| - n_outputs = 1; |
| - |
| - auto * res = gf_res_prev_mtp.get(); |
| - const auto gparams = graph_params_mtp(res, ub, mctx.get()); |
| - res->reset(); |
| - ggml_backend_sched_reset(sched_mtp.get()); |
| - |
| - auto * gf = model.build_graph(gparams); |
| - n_outputs = save_n_outputs; |
| - |
| - if (!gf) { |
| - LLAMA_LOG_ERROR("%s: failed to build MTP reserve graph\n", __func__); |
| - sched_mtp.reset(); |
| - gf_res_prev_mtp.reset(); |
| - return false; |
| - } |
| - if (!ggml_backend_sched_reserve(sched_mtp.get(), gf)) { |
| - LLAMA_LOG_ERROR("%s: failed to reserve compute buffers on sched_mtp\n", __func__); |
| - sched_mtp.reset(); |
| - gf_res_prev_mtp.reset(); |
| - return false; |
| - } |
| - // Discard the reserve graph cache so the first real call rebuilds with proper inputs. |
| - res->reset(); |
| - ggml_backend_sched_reset(sched_mtp.get()); |
| - } |
| - |
| - // opencoti F5 M6-S4 mtp (P4): the async worker thread is spawned lazily in |
| - // decode_mtp_async (only when the opt-in async path is actually taken), NOT here β so the |
| - // default in-thread sync path creates no extra thread and is runtime-identical to P3. |
| - |
| - mtp_reserved_steps = need; // reserve now covers `need` fused steps |
| - return true; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -llm_graph_result * llama_context::process_ubatch_mtp( |
| - const llama_ubatch & ubatch, |
| - llama_memory_context_i * mctx, |
| - ggml_status & ret) { |
| - GGML_ASSERT(sched_mtp && gf_res_prev_mtp); |
| - |
| - auto * res = gf_res_prev_mtp.get(); |
| - auto * gf = res->get_gf(); |
| - |
| - // graph_params_mtp reads ctx->n_outputs; for MTP it is always 1, but we set it |
| - // explicitly here in case a concurrent target decode mutates it. The MTP graph |
| - // outputs a single logits row + h_post, so n_outputs=1 is the only valid value. |
| - llm_graph_params gparams = graph_params_mtp(res, ubatch, mctx); |
| - gparams.n_outputs = 1; |
| - gparams.sched = sched_mtp.get(); |
| - |
| - if (!graph_reuse_disable && res->can_reuse(gparams)) { |
| - // sched_mtp does not use pipeline parallelism (created with pipeline=false), |
| - // so no synchronize is needed before set_inputs. |
| - } else { |
| - res->reset(); |
| - |
| - ggml_backend_sched_reset(sched_mtp.get()); |
| - ggml_backend_sched_set_eval_callback(sched_mtp.get(), cparams.cb_eval, cparams.cb_eval_user_data); |
| - |
| - gf = model.build_graph(gparams); |
| - if (!gf) { |
| - LLAMA_LOG_ERROR("%s: failed to initialize MTP graph\n", __func__); |
| - ret = GGML_STATUS_FAILED; |
| - return nullptr; |
| - } |
| - if (!ggml_backend_sched_alloc_graph(sched_mtp.get(), gf)) { |
| - LLAMA_LOG_ERROR("%s: failed to allocate MTP graph\n", __func__); |
| - ret = GGML_STATUS_ALLOC_FAILED; |
| - return nullptr; |
| - } |
| - } |
| - |
| - res->set_inputs(&ubatch); |
| - |
| - const auto status = graph_compute_mtp(res->get_gf()); |
| - |
| - if (status != GGML_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: failed to compute MTP graph, compute status: %d\n", __func__, status); |
| - ret = status; |
| - return nullptr; |
| - } |
| - |
| - ret = GGML_STATUS_SUCCESS; |
| - return res; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -ggml_status llama_context::graph_compute_mtp(ggml_cgraph * gf) { |
| - // MTP graphs are always single-token (batched=false). We mirror graph_compute()'s |
| - // threadpool/n_threads dance only for the CPU backend; the MTP head is small and |
| - // thread saturation matters less, but consistency with graph_compute() avoids |
| - // surprising backend reconfigurations between target and MTP runs. |
| - // opencoti P2 adaptation: the fork guards this block with backend_cfg_mu (an async-worker |
| - // mutex). The sync-only P2 path runs single-threaded, so no lock is needed. |
| - const int n_threads = cparams.n_threads; |
| - ggml_threadpool_t tp = threadpool; |
| - |
| - if (backend_cpu != nullptr) { |
| - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| - auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) |
| - ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); |
| - if (set_threadpool_fn) { |
| - set_threadpool_fn(backend_cpu, tp); |
| - } |
| - } |
| - for (const auto & set_n_threads_fn : set_n_threads_fns) { |
| - set_n_threads_fn.second(set_n_threads_fn.first, n_threads); |
| - } |
| - |
| - auto status = ggml_backend_sched_graph_compute_async(sched_mtp.get(), gf); |
| - if (status != GGML_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async (MTP) failed with %d\n", |
| - __func__, status); |
| - } |
| - return status; |
| -} |
| - |
| llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { |
| if (mctx && !mctx->apply()) { |
| LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__); |
| @@ -2093,14 +1880,6 @@ int llama_context::decode(const llama_batch & batch_inp) { |
| |
| bool did_optimize = false; |
| |
| - // opencoti F5 M6-S4 mtp (P4): drain any in-flight MTP draft before the target mutates its |
| - // KV below (memory_update's pending shifts/copies, then init_batch's slot apply β the S1 |
| - // seq_cp flush / M7 window-slide surfaces of R-S4b). The async worker cross-reads this |
| - // sequence's KV read-only; draining first lets that read retire against stable KV. Inert in |
| - // the current sync-facade integration (nothing is in flight here) and a single pointer |
| - // check when no MTP assistant is loaded. |
| - mtp_drain_before_mutate(); |
| - |
| // handle any pending shifts/copies |
| memory_update(false); |
| |
| @@ -2684,368 +2463,11 @@ ggml_cgraph * llama_context::graph_reserve( |
| return gf; |
| } |
| |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -llm_graph_params llama_context::graph_params_mtp( |
| - llm_graph_result * res, |
| - const llama_ubatch & ubatch, |
| - const llama_memory_context_i * mctx) const { |
| - const llama_model * mtp = model.mtp_assistant.get(); |
| - GGML_ASSERT(mtp); |
| - |
| - return { |
| - /*.arch =*/ mtp->arch, |
| - /*.hparams =*/ mtp->hparams, |
| - /*.cparams =*/ cparams, |
| - /*.ubatch =*/ ubatch, |
| - /*.gtype =*/ LLM_GRAPH_TYPE_MTP, |
| - /*.sched =*/ sched.get(), |
| - /*.backend_cpu =*/ backend_cpu, |
| - /*.cvec =*/ cvec.get(), |
| - /*.loras =*/ loras.get(), |
| - /*.mctx =*/ mctx, |
| - /*.cross =*/ &cross, |
| - /*.samplers =*/ sampling.samplers, |
| - /*.n_outputs =*/ n_outputs, |
| - /*.cb =*/ graph_get_cb(), |
| - /*.res =*/ res, |
| - /*.n_mtp_steps =*/ mtp_fused_steps, // opencoti F5 M6-S4 mtp (P4 fused) |
| - }; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P2 inert): synchronous facade only; async worker is P4. |
| -int32_t llama_context::decode_mtp( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last) { |
| - // opencoti F5 M6-S4 mtp (P4): sync facade over the async worker. The worker contract streams |
| - // no per-step logits, so any caller needing out_logits (the logit-equiv harness) takes the |
| - // in-thread decode_mtp_sync path; the normal draft path (out_logits==NULL) goes asyncβwait. |
| - // With no target/draft overlap this is byte-identical to the P3 sync path β the worker runs |
| - // the same N-step loop off-thread while we block in decode_mtp_wait. |
| - if (out_logits) { |
| - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, |
| - out_drafts, out_logits, out_h_prev_last); |
| - } |
| - // opencoti F5 M6-S4 mtp (P4 fused): the live draft path (out_logits==NULL) defaults to the |
| - // fused single-graph draft (decode_mtp_fused) β ONE launch+sync+D2H for the whole block, the |
| - // throughput fix. OPENCOTI_MTP_FUSED=0 forces the per-step sequential path for A/B (the fused |
| - // drafts are gated byte-identical to it). n_steps<=1 has nothing to fuse β sequential. |
| - static const bool fused_enabled = [] { |
| - const char * e = getenv("OPENCOTI_MTP_FUSED"); |
| - return !(e && e[0] == '0'); |
| - }(); |
| - if (fused_enabled && n_steps > 1) { |
| - return decode_mtp_fused(seq_id, attn_pos, last_token, h_prev, n_steps, |
| - out_drafts, out_h_prev_last); |
| - } |
| - // opencoti F5 M6-S4 mtp (P4): the async worker path is OPT-IN (OPENCOTI_MTP_ASYNC=1). |
| - // Default = the proven in-thread sync path. Rationale: in sync-facade (depth-1) the worker |
| - // gives NO throughput benefit β real depth-2 overlap is deferred (limited by the h-prev |
| - // dependency; even the AtomicBot fork ships depth-1) β AND the worker-thread CUDA execution |
| - // currently segfaults under the cosmocc DSO at the first MTP draft (bug-405). Gated off |
| - // until both a measured depth-2 win and that crash are resolved; the plumbing stays for it. |
| - static const bool async_enabled = [] { |
| - const char * e = getenv("OPENCOTI_MTP_ASYNC"); |
| - return e && e[0] == '1'; |
| - }(); |
| - if (!async_enabled) { |
| - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, |
| - out_drafts, out_logits, out_h_prev_last); |
| - } |
| - const int32_t rc = decode_mtp_async(seq_id, attn_pos, last_token, h_prev, n_steps); |
| - if (rc != 0) { |
| - return rc; |
| - } |
| - return decode_mtp_wait(out_drafts, out_h_prev_last); |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -int32_t llama_context::decode_mtp_sync( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last) { |
| - if (!model.mtp_assistant) { |
| - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); |
| - return -1; |
| - } |
| - if (!memory) { |
| - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); |
| - return -2; |
| - } |
| - auto * kv_iswa = dynamic_cast<llama_kv_cache_iswa *>(memory.get()); |
| - if (!kv_iswa) { |
| - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); |
| - return -3; |
| - } |
| - |
| - // opencoti F5 M6-S4 mtp (P4 fused): the sequential path always builds the single-step graph, |
| - // even if a prior decode_mtp_fused left mtp_fused_steps >1. (The larger reserve still fits.) |
| - mtp_fused_steps = 1; |
| - |
| - if (!ensure_sched_mtp()) { |
| - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); |
| - return -8; |
| - } |
| - |
| - const int32_t n_vocab = model.vocab.n_tokens(); |
| - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; |
| - if (n_bb == 0) { |
| - LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); |
| - return -4; |
| - } |
| - |
| - auto data = std::make_shared<llama_ubatch::data_t>(); |
| - data->token.resize(1); |
| - data->embd.resize(n_bb); |
| - data->pos.resize(1); |
| - data->n_seq_id.resize(1); |
| - data->seq_id.resize(1); |
| - data->seq_id_data.resize(1); |
| - data->output.resize(1); |
| - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); |
| - data->seq_id_unq.push_back(seq_id); |
| - data->seq_idx[(size_t) seq_id] = 0; |
| - |
| - llama_ubatch ub{}; |
| - ub.b_equal_seqs = 1; |
| - ub.n_tokens = 1; |
| - ub.n_seq_tokens = 1; |
| - ub.n_seqs = 1; |
| - ub.n_seqs_unq = 1; |
| - ub.n_pos = 1; |
| - ub.token = data->token.data(); |
| - ub.embd = data->embd.data(); |
| - ub.pos = data->pos.data(); |
| - ub.n_seq_id = data->n_seq_id.data(); |
| - ub.seq_id = data->seq_id.data(); |
| - ub.seq_id_unq = data->seq_id_unq.data(); |
| - ub.seq_idx = data->seq_idx.data(); |
| - ub.output = data->output.data(); |
| - ub.data = data; |
| - |
| - data->n_seq_id[0] = 1; |
| - data->seq_id_data[0] = seq_id; |
| - data->seq_id[0] = &data->seq_id_data[0]; |
| - // opencoti F5 M6-S4 mtp: the single MTP token IS the output (we read its argmax/logits/h_post |
| - // back). It MUST be flagged output=1 so build_inp_out_ids selects its row β and so the live |
| - // graph topology matches the reserve graph (ensure_sched_mtp also uses output=1). A 0 here |
| - // produced a divergent graph whose output-selection tensor was left with a null backend |
| - // buffer, asserting in compute_splits (bug-404). Matches the AtomicBot fork. |
| - data->output[0] = 1; |
| - |
| - if (n_steps <= 0) { |
| - return 0; |
| - } |
| - |
| - // Sequential MTP draft on the dedicated sched_mtp: per step run a fresh single-token |
| - // graph; each step's argmax feeds next step's last_token; h_post -> next step's h_prev. |
| - for (int32_t k = 0; k < n_steps; ++k) { |
| - data->token[0] = last_token; |
| - // bug-858: the gemma4-assistant (is_mem_shared) drafts ALL steps from the SAME query |
| - // position β the head is trained to predict multiple future tokens from one position, with |
| - // the recurrence carried by the h_prev hidden chain (NOT the RoPE angle). Incrementing the |
| - // position per step (attn_pos+1+k) hands the head a mistrained RoPE angle from step 1 on, |
| - // degrading step-1+ drafts (aggregate accept ~0.69 vs upstream 0.79). Match b9859 draft_mtp |
| - // is_mem_shared: common_batch_add(batch, id, dp.n_past, ...) β constant dp.n_past = attn_pos+1. |
| - data->pos[0] = attn_pos + 1; |
| - std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); |
| - |
| - llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); |
| - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); |
| - return -5; |
| - } |
| - |
| - ggml_status status = GGML_STATUS_SUCCESS; |
| - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); |
| - if (!res || status != GGML_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); |
| - return -6; |
| - } |
| - |
| - ggml_backend_sched_synchronize(sched_mtp.get()); |
| - |
| - ggml_tensor * t_arg = res->get_argmax(); |
| - GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); |
| - |
| - int32_t best_i32 = 0; |
| - ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); |
| - out_drafts[k] = (llama_token) best_i32; |
| - |
| - if (out_logits) { |
| - ggml_tensor * t_logits = res->get_logits(); |
| - GGML_ASSERT(t_logits); |
| - ggml_backend_tensor_get(t_logits, out_logits + (int64_t) k * n_vocab, |
| - 0, (size_t) n_vocab * sizeof(float)); |
| - } |
| - |
| - ggml_tensor * t_post = res->get_embd(); |
| - GGML_ASSERT(t_post); |
| - ggml_backend_tensor_get(t_post, h_prev, 0, n_bb * sizeof(float)); |
| - |
| - last_token = (llama_token) best_i32; |
| - } |
| - |
| - if (out_h_prev_last) { |
| - std::memcpy(out_h_prev_last, h_prev, n_bb * sizeof(float)); |
| - } |
| - |
| - return 0; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P4 fused): draft the whole N-token block in ONE graph. The MTP head |
| -// writes no KV and never self-attends across draft steps, so the autoregressive chain unrolls |
| -// in-graph: each step's on-device argmax feeds the next step's token (a get_rows index β already |
| -// how build_one_step routes top_k/flat_ids) and each step's backbone feeds the next h_prev. |
| -// Result: ONE launch + ONE ggml_backend_sched_synchronize + ONE D2H of the N drafted tokens, |
| -// instead of decode_mtp_sync's N synced single-token graphs (the throughput fix). |
| -// |
| -// One mask shared across steps (built from step 0's position). For FULL-attn target layers and |
| -// for SWA layers whose window covers the whole prefix (attn_pos+1 <= n_swa β the short-context / |
| -// throughput-bench regime) this is EXACT: every step admits all target cells, so fused drafts are |
| -// byte-identical to the sequential path. For SWA layers at long context the true per-step window |
| -// shifts by k boundary cells, so steps 1..N-1 may attend up to N stale far-edge cells. That is |
| -// output-safe β the target verify pass rejects any mismatched draft, so the emitted stream is |
| -// unchanged; only the accept rate can drift. (If long-ctx accept regresses vs sequential, the fix |
| -// is a per-step mask slice β deferred until measured. See buglog bug-420.) |
| -int32_t llama_context::decode_mtp_fused( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_h_prev_last) { |
| - if (!model.mtp_assistant) { |
| - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); |
| - return -1; |
| - } |
| - if (!memory) { |
| - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); |
| - return -2; |
| - } |
| - auto * kv_iswa = dynamic_cast<llama_kv_cache_iswa *>(memory.get()); |
| - if (!kv_iswa) { |
| - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); |
| - return -3; |
| - } |
| - if (n_steps <= 0) { |
| - return 0; |
| - } |
| - // A single step has nothing to fuse β defer to the proven sequential path. |
| - if (n_steps == 1) { |
| - return decode_mtp_sync(seq_id, attn_pos, last_token, h_prev, n_steps, |
| - out_drafts, /*out_logits=*/nullptr, out_h_prev_last); |
| - } |
| - |
| - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; |
| - if (n_bb == 0) { |
| - LLAMA_LOG_ERROR("%s: assistant missing embedding_length_out (backbone width) metadata\n", __func__); |
| - return -4; |
| - } |
| - |
| - // Build/reserve a fused n_steps graph. |
| - mtp_fused_steps = n_steps; |
| - if (!ensure_sched_mtp()) { |
| - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); |
| - return -8; |
| - } |
| - |
| - // n_tokens stays 1 (each fused step processes a single token); pos[] carries the N per-step |
| - // query positions consumed by the wrapper's inp_pos_steps (set_input). |
| - auto data = std::make_shared<llama_ubatch::data_t>(); |
| - data->token.resize(1); |
| - data->embd.resize(n_bb); |
| - data->pos.resize(n_steps); |
| - data->n_seq_id.resize(1); |
| - data->seq_id.resize(1); |
| - data->seq_id_data.resize(1); |
| - data->output.resize(1); |
| - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); |
| - data->seq_id_unq.push_back(seq_id); |
| - data->seq_idx[(size_t) seq_id] = 0; |
| - |
| - llama_ubatch ub{}; |
| - ub.b_equal_seqs = 1; |
| - ub.n_tokens = 1; |
| - ub.n_seq_tokens = 1; |
| - ub.n_seqs = 1; |
| - ub.n_seqs_unq = 1; |
| - ub.n_pos = (uint32_t) n_steps; |
| - ub.token = data->token.data(); |
| - ub.embd = data->embd.data(); |
| - ub.pos = data->pos.data(); |
| - ub.n_seq_id = data->n_seq_id.data(); |
| - ub.seq_id = data->seq_id.data(); |
| - ub.seq_id_unq = data->seq_id_unq.data(); |
| - ub.seq_idx = data->seq_idx.data(); |
| - ub.output = data->output.data(); |
| - ub.data = data; |
| - |
| - data->n_seq_id[0] = 1; |
| - data->seq_id_data[0] = seq_id; |
| - data->seq_id[0] = &data->seq_id_data[0]; |
| - data->output[0] = 1; |
| - |
| - data->token[0] = last_token; |
| - std::memcpy(data->embd.data(), h_prev, n_bb * sizeof(float)); |
| - // bug-858: constant query position for ALL draft steps (see decode_mtp_sync note). The |
| - // gemma4-assistant (is_mem_shared) is trained to predict every future draft token from the |
| - // SAME position (dp.n_past = attn_pos+1); recurrence lives in the h_prev chain, not the RoPE |
| - // angle. The old attn_pos+1+k mistrained step-1+ RoPE -> accept 0.69 vs upstream 0.79. |
| - for (int32_t k = 0; k < n_steps; ++k) { |
| - data->pos[k] = attn_pos + 1; |
| - } |
| - |
| - llama_memory_context_ptr mctx = kv_iswa->init_mtp(seq_id, ub); |
| - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: init_mtp failed\n", __func__); |
| - return -5; |
| - } |
| - |
| - ggml_status status = GGML_STATUS_SUCCESS; |
| - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); |
| - if (!res || status != GGML_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: fused MTP graph failed (status %d)\n", __func__, (int) status); |
| - return -6; |
| - } |
| - |
| - ggml_backend_sched_synchronize(sched_mtp.get()); |
| - |
| - ggml_tensor * t_arg = res->get_argmax(); |
| - GGML_ASSERT(t_arg && "fused MTP graph must publish the in-graph argmax block"); |
| - GGML_ASSERT(t_arg->ne[0] == (int64_t) n_steps && "fused MTP argmax must be I32[n_steps]"); |
| - |
| - std::vector<int32_t> drafts((size_t) n_steps); |
| - ggml_backend_tensor_get(t_arg, drafts.data(), 0, (size_t) n_steps * sizeof(int32_t)); |
| - for (int32_t k = 0; k < n_steps; ++k) { |
| - out_drafts[k] = (llama_token) drafts[(size_t) k]; |
| - } |
| - |
| - if (out_h_prev_last) { |
| - ggml_tensor * t_post = res->get_embd(); |
| - GGML_ASSERT(t_post); |
| - ggml_backend_tensor_get(t_post, out_h_prev_last, 0, n_bb * sizeof(float)); |
| - } |
| - |
| - return 0; |
| -} |
| - |
| // opencoti fused-NextN (#590 / bug-858): fuse the N per-step NextN draft decodes into ONE graph on |
| // the draft context (ctx_dft, type LLAMA_CONTEXT_TYPE_MTP β LLM_GRAPH_TYPE_DECODER_MTP). This is the |
| // throughput fix: the un-fused path pays a host sync per draft step; here one graph emits N greedy |
| -// drafts with a single sync. Twin of decode_mtp_fused (Gemma assistant) but for the UNIFIED cache β |
| -// so no mtp_assistant / sched_mtp / kv_iswa. The N fused steps read-only cross-attend the frozen |
| +// drafts with a single sync, for the UNIFIED cache β |
| +// so no mtp_assistant / kv_iswa involvement. The N fused steps read-only cross-attend the frozen |
| // prefix (build_attn_readonly_nextn, Option A); recurrence is carried by the on-device hidden chain. |
| // GREEDY-ONLY: no p_min early-stop (the caller reconciles draft length). See |
| // docs/features/fused_nextn_mtp.md. Returns 0 on success, negative on error. |
| @@ -3062,7 +2484,7 @@ int32_t llama_context::decode_mtp_fused_nextn( |
| return -2; |
| } |
| // NextN self-spec targets are full-attention β unified cache. (iSWA NextN is not a thing; |
| - // the Gemma assistant path is decode_mtp_fused.) |
| + // the Gemma assistant runs as a dual context via draft_mtp, not through this driver.) |
| auto * kv = dynamic_cast<llama_kv_cache *>(memory.get()); |
| if (!kv) { |
| LLAMA_LOG_ERROR("%s: fused NextN requires a unified llama_kv_cache\n", __func__); |
| @@ -3133,7 +2555,7 @@ int32_t llama_context::decode_mtp_fused_nextn( |
| } |
| |
| // Build/compute the fused N-step graph on the NORMAL sched (graph_params sets n_mtp_steps=N for |
| - // DECODER_MTP). Inlined like process_ubatch_mtp β deliberately NOT process_ubatch, whose apply() |
| + // DECODER_MTP). Inlined graph processing β deliberately NOT process_ubatch, whose apply() |
| // would overwrite the pmax prefix cell. |
| mtp_fused_steps = n_steps; |
| |
| @@ -3187,7 +2609,7 @@ int32_t llama_context::decode_mtp_fused_nextn( |
| } |
| res->set_inputs(&ub); |
| |
| - // Compute on the dedicated sched (mirror graph_compute_mtp's CPU-threadpool dance). |
| + // Compute on the dedicated sched (with the CPU-threadpool attach/set dance). |
| if (backend_cpu != nullptr) { |
| auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| auto * set_tp_fn = (decltype(ggml_backend_cpu_set_threadpool) *) |
| @@ -3227,224 +2649,6 @@ int32_t llama_context::decode_mtp_fused_nextn( |
| return 0; |
| } |
| |
| -// opencoti F5 M6-S4 mtp (P4): submit one async MTP draft request to the worker. At most one |
| -// in-flight request per context (returns -7 if a prior request was not yet waited). |
| -int32_t llama_context::decode_mtp_async( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - const float * h_prev, |
| - int32_t n_steps) { |
| - if (!model.mtp_assistant) { |
| - LLAMA_LOG_ERROR("%s: no MTP assistant loaded (use llama_model_load_mtp_from_file)\n", __func__); |
| - return -1; |
| - } |
| - if (!memory) { |
| - LLAMA_LOG_ERROR("%s: context has no KV memory\n", __func__); |
| - return -2; |
| - } |
| - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; |
| - if (n_bb == 0 || !h_prev || n_steps <= 0) { |
| - LLAMA_LOG_ERROR("%s: invalid arguments (n_bb=%u, h_prev=%p, n_steps=%d)\n", |
| - __func__, n_bb, (const void *) h_prev, n_steps); |
| - return -4; |
| - } |
| - if (!ensure_sched_mtp()) { |
| - LLAMA_LOG_ERROR("%s: failed to initialize MTP scheduler\n", __func__); |
| - return -8; |
| - } |
| - if (!mtp_worker.joinable()) { |
| - mtp_worker = std::thread(&llama_context::mtp_worker_loop, this); |
| - } |
| - |
| - { |
| - std::unique_lock<std::mutex> lk(mtp_mu); |
| - if (mtp_pending.has_value() || mtp_in_flight || mtp_completed.has_value()) { |
| - LLAMA_LOG_ERROR("%s: previous MTP request not yet waited (pending=%d in_flight=%d completed=%d)\n", |
| - __func__, (int) mtp_pending.has_value(), (int) mtp_in_flight, |
| - (int) mtp_completed.has_value()); |
| - return -7; |
| - } |
| - mtp_request req; |
| - req.seq_id = seq_id; |
| - req.attn_pos = attn_pos; |
| - req.last_token = last_token; |
| - req.n_steps = n_steps; |
| - req.h_prev.assign(h_prev, h_prev + n_bb); |
| - mtp_pending = std::move(req); |
| - } |
| - mtp_cv_request.notify_one(); |
| - return 0; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P4): block until the in-flight request completes; copy drafts + last |
| -// hidden out. Consumes mtp_completed (paired 1:1 with a prior decode_mtp_async). |
| -int32_t llama_context::decode_mtp_wait( |
| - llama_token * out_drafts, |
| - float * out_h_prev_last) { |
| - std::unique_lock<std::mutex> lk(mtp_mu); |
| - mtp_cv_response.wait(lk, [this] { |
| - return mtp_completed.has_value() || (!mtp_in_flight && !mtp_pending.has_value()); |
| - }); |
| - if (!mtp_completed.has_value()) { |
| - LLAMA_LOG_ERROR("%s: no in-flight MTP request to wait on\n", __func__); |
| - return -7; |
| - } |
| - mtp_response resp = std::move(*mtp_completed); |
| - mtp_completed.reset(); |
| - lk.unlock(); |
| - |
| - if (resp.status != 0) { |
| - return resp.status; |
| - } |
| - if (out_drafts && !resp.drafts.empty()) { |
| - std::memcpy(out_drafts, resp.drafts.data(), resp.drafts.size() * sizeof(llama_token)); |
| - } |
| - if (out_h_prev_last && !resp.h_prev_last.empty()) { |
| - std::memcpy(out_h_prev_last, resp.h_prev_last.data(), resp.h_prev_last.size() * sizeof(float)); |
| - } |
| - return 0; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P4): worker-side N-step draft. Identical math to decode_mtp_sync's |
| -// loop. NOTE output[0]=1 (NOT the fork's 0): process_ubatch_mtp hard-sets n_outputs=1, so the |
| -// single ubatch row MUST be flagged output β a 0 here reintroduces the null-out-ids abort |
| -// (bug-404). Runs entirely on sched_mtp; the async contract streams no per-step logits. |
| -int32_t llama_context::decode_mtp_run(const mtp_request & req, mtp_response & resp) { |
| - auto * kv_iswa = dynamic_cast<llama_kv_cache_iswa *>(memory.get()); |
| - if (!kv_iswa) { |
| - LLAMA_LOG_ERROR("%s: MTP requires llama_kv_cache_iswa memory (Gemma 4 target)\n", __func__); |
| - return -3; |
| - } |
| - const uint32_t n_bb = model.mtp_assistant->hparams.n_embd_out_impl; |
| - |
| - auto data = std::make_shared<llama_ubatch::data_t>(); |
| - data->token.resize(1); |
| - data->embd.resize(n_bb); |
| - data->pos.resize(1); |
| - data->n_seq_id.resize(1); |
| - data->seq_id.resize(1); |
| - data->seq_id_data.resize(1); |
| - data->output.resize(1); |
| - data->seq_idx.resize(LLAMA_MAX_SEQ, -1); |
| - data->seq_id_unq.push_back(req.seq_id); |
| - data->seq_idx[(size_t) req.seq_id] = 0; |
| - |
| - llama_ubatch ub{}; |
| - ub.b_equal_seqs = 1; |
| - ub.n_tokens = 1; |
| - ub.n_seq_tokens = 1; |
| - ub.n_seqs = 1; |
| - ub.n_seqs_unq = 1; |
| - ub.n_pos = 1; |
| - ub.token = data->token.data(); |
| - ub.embd = data->embd.data(); |
| - ub.pos = data->pos.data(); |
| - ub.n_seq_id = data->n_seq_id.data(); |
| - ub.seq_id = data->seq_id.data(); |
| - ub.seq_id_unq = data->seq_id_unq.data(); |
| - ub.seq_idx = data->seq_idx.data(); |
| - ub.output = data->output.data(); |
| - ub.data = data; |
| - |
| - data->n_seq_id[0] = 1; |
| - data->seq_id_data[0] = req.seq_id; |
| - data->seq_id[0] = &data->seq_id_data[0]; |
| - data->output[0] = 1; // bug-404: must match process_ubatch_mtp's n_outputs=1 |
| - |
| - std::vector<float> h(req.h_prev); |
| - llama_token last_token = req.last_token; |
| - resp.drafts.assign((size_t) req.n_steps, 0); |
| - |
| - for (int32_t k = 0; k < req.n_steps; ++k) { |
| - data->token[0] = last_token; |
| - data->pos[0] = req.attn_pos + 1 + (llama_pos) k; |
| - std::memcpy(data->embd.data(), h.data(), n_bb * sizeof(float)); |
| - |
| - llama_memory_context_ptr mctx = kv_iswa->init_mtp(req.seq_id, ub); |
| - if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: init_mtp failed at step %d\n", __func__, k); |
| - return -5; |
| - } |
| - |
| - ggml_status status = GGML_STATUS_SUCCESS; |
| - llm_graph_result * res = process_ubatch_mtp(ub, mctx.get(), status); |
| - if (!res || status != GGML_STATUS_SUCCESS) { |
| - LLAMA_LOG_ERROR("%s: MTP graph failed at step %d (status %d)\n", __func__, k, (int) status); |
| - return -6; |
| - } |
| - |
| - ggml_backend_sched_synchronize(sched_mtp.get()); |
| - |
| - ggml_tensor * t_arg = res->get_argmax(); |
| - GGML_ASSERT(t_arg && "MTP graph must publish in-graph argmax tensor"); |
| - int32_t best_i32 = 0; |
| - ggml_backend_tensor_get(t_arg, &best_i32, 0, sizeof(int32_t)); |
| - last_token = (llama_token) best_i32; |
| - resp.drafts[(size_t) k] = last_token; |
| - |
| - ggml_tensor * t_post = res->get_embd(); |
| - GGML_ASSERT(t_post); |
| - ggml_backend_tensor_get(t_post, h.data(), 0, n_bb * sizeof(float)); |
| - } |
| - |
| - resp.h_prev_last = std::move(h); |
| - return 0; |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P4): producer/consumer loop. Waits for a request, runs it on |
| -// sched_mtp, publishes the response. Exits when mtp_worker_stop is set and no request pends. |
| -void llama_context::mtp_worker_loop() { |
| - for (;;) { |
| - mtp_request req; |
| - { |
| - std::unique_lock<std::mutex> lk(mtp_mu); |
| - mtp_cv_request.wait(lk, [this] { |
| - return mtp_worker_stop.load(std::memory_order_acquire) || mtp_pending.has_value(); |
| - }); |
| - if (mtp_worker_stop.load(std::memory_order_acquire) && !mtp_pending.has_value()) { |
| - return; |
| - } |
| - req = std::move(*mtp_pending); |
| - mtp_pending.reset(); |
| - mtp_in_flight = true; |
| - } |
| - |
| - mtp_response resp; |
| - resp.status = decode_mtp_run(req, resp); |
| - |
| - { |
| - std::lock_guard<std::mutex> lk(mtp_mu); |
| - mtp_in_flight = false; |
| - mtp_completed = std::move(resp); |
| - } |
| - mtp_cv_response.notify_one(); |
| - } |
| -} |
| - |
| -// opencoti F5 M6-S4 mtp (P4): R-S4b drain. Wait for the worker to go idle (its read-only KV |
| -// read retired), then sync sched_mtp β WITHOUT consuming mtp_completed (the spec loop's _wait |
| -// still needs the drafts). Called at decode() top before any KV mutation. Inert when no |
| -// assistant is loaded or nothing is in flight (drained==false β no extra sched sync). |
| -void llama_context::mtp_drain_before_mutate() { |
| - if (!model.mtp_assistant) { |
| - return; |
| - } |
| - bool drained = false; |
| - { |
| - std::unique_lock<std::mutex> lk(mtp_mu); |
| - if (mtp_pending.has_value() || mtp_in_flight) { |
| - mtp_cv_response.wait(lk, [this] { |
| - return !mtp_in_flight && !mtp_pending.has_value(); |
| - }); |
| - drained = true; |
| - } |
| - } |
| - if (drained && sched_mtp) { |
| - ggml_backend_sched_synchronize(sched_mtp.get()); |
| - } |
| -} |
| - |
| llm_graph_params llama_context::graph_params( |
| llm_graph_result * res, |
| const llama_ubatch & ubatch, |
| @@ -5177,24 +4381,6 @@ int32_t llama_decode( |
| return ret; |
| } |
| |
| -// opencoti F5 M6-S4 mtp (P2 inert) |
| -int32_t llama_decode_mtp( |
| - llama_context * ctx, |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last) { |
| - if (!ctx) { |
| - LLAMA_LOG_ERROR("%s: ctx is NULL\n", __func__); |
| - return -1; |
| - } |
| - return ctx->decode_mtp(seq_id, attn_pos, last_token, h_prev, n_steps, out_drafts, out_logits, out_h_prev_last); |
| -} |
| - |
| int32_t llama_decode_mtp_fused_nextn( |
| llama_context * ctx, |
| llama_seq_id seq_id, |
| |
| |
| |
| |
| @@ -147,48 +147,8 @@ struct llama_context { |
| llama_memory_context_i * mctx, |
| ggml_status & ret); |
| |
| - // opencoti F5 M6-S4 mtp (P2 inert): synchronous Gemma4 MTP draft machinery. |
| - llm_graph_params graph_params_mtp( |
| - llm_graph_result * res, |
| - const llama_ubatch & ubatch, |
| - const llama_memory_context_i * mctx) const; |
| - |
| - int32_t decode_mtp( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last); |
| - |
| - int32_t decode_mtp_sync( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_logits, |
| - float * out_h_prev_last); |
| - |
| - // opencoti F5 M6-S4 mtp (P4 fused): the throughput path. Unrolls all n_steps draft steps |
| - // into ONE graph (each step's on-device argmax/backbone feeds the next), so the whole draft |
| - // block costs ONE graph launch + ONE GPU sync + N small D2H reads instead of n_steps synced |
| - // single-token graphs. Live draft only (out_logits is implicitly NULL); the per-step-logits |
| - // verification path stays on decode_mtp_sync. |
| - int32_t decode_mtp_fused( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - float * h_prev, |
| - int32_t n_steps, |
| - llama_token * out_drafts, |
| - float * out_h_prev_last); |
| - |
| // opencoti fused-NextN (#590): fused N-step greedy draft on a DECODER_MTP (Qwen NextN) draft |
| - // context using the unified cache. Twin of decode_mtp_fused minus mtp_assistant/sched_mtp/iSWA. |
| + // context using the unified cache (the only in-context MTP draft driver). |
| int32_t decode_mtp_fused_nextn( |
| llama_seq_id seq_id, |
| llama_pos attn_pos, |
| @@ -198,25 +158,6 @@ struct llama_context { |
| llama_token * out_drafts, |
| float * out_h_prev_last); |
| |
| - // opencoti F5 M6-S4 mtp (P4): async MTP draft pipeline. decode_mtp() (above) is a sync |
| - // facade β out_logits!=NULL keeps the in-thread decode_mtp_sync path (the worker streams |
| - // no per-step logits); otherwise it submits via decode_mtp_async and blocks in |
| - // decode_mtp_wait. With no target/draft overlap yet this is behaviourally identical to the |
| - // P3 sync path; the worker + drain guard are the foundation for future depth-2 (R-S4b). |
| - int32_t decode_mtp_async( |
| - llama_seq_id seq_id, |
| - llama_pos attn_pos, |
| - llama_token last_token, |
| - const float * h_prev, |
| - int32_t n_steps); |
| - int32_t decode_mtp_wait( |
| - llama_token * out_drafts, |
| - float * out_h_prev_last); |
| - // Drain any in-flight MTP draft (wait for the worker's read-only KV read to retire) before |
| - // the target mutates its KV. Does NOT consume the completed result. Inert when no MTP |
| - // assistant is loaded or nothing is in flight (the common case). |
| - void mtp_drain_before_mutate(); |
| - |
| int encode(const llama_batch & batch_inp); |
| int decode(const llama_batch & batch_inp); |
| |
| @@ -333,14 +274,6 @@ private: |
| const llama_memory_context_i * mctx, |
| llm_graph_type gtype) const; |
| |
| - // opencoti F5 M6-S4 mtp (P2 inert): dedicated single-token MTP sched + reuse cache. |
| - bool ensure_sched_mtp(); |
| - llm_graph_result * process_ubatch_mtp( |
| - const llama_ubatch & ubatch, |
| - llama_memory_context_i * mctx, |
| - ggml_status & ret); |
| - ggml_status graph_compute_mtp(ggml_cgraph * gf); |
| - |
| llm_graph_cb graph_get_cb() const; |
| |
| // TODO: read/write lora adapters and cvec |
| @@ -440,58 +373,22 @@ private: |
| llm_graph_result_ptr gf_res_prev; |
| llm_graph_result_ptr gf_res_reserve; |
| |
| - // opencoti F5 M6-S4 mtp (P2 inert): dedicated scheduler so the MTP draft graph can be |
| - // encoded without contending with the target's sched. gf_res_prev_mtp keeps the reusable |
| - // single-token MTP graph result across steps of one decode_mtp call. |
| - ggml_backend_sched_ptr sched_mtp; |
| - llm_graph_result_ptr gf_res_prev_mtp; |
| - |
| // opencoti #590/bug-858: DEDICATED scheduler + result for the fused NextN self-spec draft |
| // (decode_mtp_fused_nextn). It must NOT share the main sched/gf_res_prev with target decode: |
| // every interleaved target decode RESETS the main sched (reallocating compute buffers) and |
| // gf_res_prev, so a fused graph parked there can never satisfy can_reuse AND its tensor memory is |
| // clobbered β 100% rebuild + CUDA-recapture per draft round (measured: build 0.5ms + comp-launch |
| - // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched (like the Gemma assistant's |
| - // sched_mtp, but ensure_sched_mtp is assistant-only: needs mtp_assistant + iswa cache) isolates the |
| + // 1.5ms, rebuilds=N/N) AND garbage argmax when reused. A dedicated sched isolates the |
| // fused graph so can_reuse compares fused-vs-fused (stable within a 256-cell n_kv bucket) and the |
| // ggml graph + CUDA capture persist across rounds. NextN and Gemma-assistant are mutually exclusive. |
| ggml_backend_sched_ptr sched_nextn; |
| llm_graph_result_ptr gf_res_prev_nextn; |
| int32_t nextn_reserved_steps = 0; |
| |
| - // opencoti F5 M6-S4 mtp (P4 fused): number of draft steps to unroll into the MTP graph for |
| - // the NEXT build (decode_mtp_fused sets it to n_steps; the sequential/reserve paths leave it |
| - // at 1). mtp_reserved_steps records how many steps the current sched_mtp reserve covers, so |
| - // ensure_sched_mtp only re-reserves when a larger fused block is requested. |
| + // opencoti #590 fused NextN: number of draft steps to unroll into the MTP graph for the |
| + // NEXT build (decode_mtp_fused_nextn sets it to n_steps around its build; the normal |
| + // decode/reserve paths leave it at 1 so graph_params stays single-step). |
| int32_t mtp_fused_steps = 1; |
| - int32_t mtp_reserved_steps = 0; |
| - |
| - // opencoti F5 M6-S4 mtp (P4): async MTP draft worker + single-slot request/response queue. |
| - // At most one in-flight request per context. The worker runs decode_mtp_run on sched_mtp |
| - // while the caller blocks in decode_mtp_wait; there is no target/draft overlap yet (sync |
| - // facade), so the worker never races the target's sched. The destructor joins the worker. |
| - struct mtp_request { |
| - llama_seq_id seq_id = 0; |
| - llama_pos attn_pos = 0; |
| - llama_token last_token = 0; |
| - std::vector<float> h_prev; |
| - int32_t n_steps = 0; |
| - }; |
| - struct mtp_response { |
| - int32_t status = 0; |
| - std::vector<llama_token> drafts; |
| - std::vector<float> h_prev_last; |
| - }; |
| - std::thread mtp_worker; |
| - std::atomic<bool> mtp_worker_stop{false}; |
| - std::mutex mtp_mu; |
| - std::condition_variable mtp_cv_request; |
| - std::condition_variable mtp_cv_response; |
| - std::optional<mtp_request> mtp_pending; // submitted, not yet picked up |
| - bool mtp_in_flight = false; // worker is processing |
| - std::optional<mtp_response> mtp_completed; // worker finished, awaiting _wait |
| - int32_t decode_mtp_run(const mtp_request & req, mtp_response & resp); // worker-side N-step loop |
| - void mtp_worker_loop(); |
| |
| // host buffer for the model output (logits and embeddings) |
| ggml_backend_buffer_ptr buf_output; |
| |
| |
| |
| |
| @@ -124,7 +124,7 @@ void llm_graph_input_mtp::set_input(const llama_ubatch * ubatch) { |
| ggml_backend_tensor_set(inp_h_prev, ubatch->embd, 0, n_bb * sizeof(float)); |
| |
| // opencoti F5 M6-S4 mtp (P4 fused): step k's RoPE query position = ubatch->pos[k] |
| - // (= attn_pos + 1 + k, filled by decode_mtp_fused). Single-step path leaves this empty. |
| + // (= attn_pos + 1 + k, filled by decode_mtp_fused_nextn). Single-step path leaves this empty. |
| // bug-870: mrope archs (qwen35) size each step's pos tensor I32[4] β fill the M-RoPE |
| // text-token layout [p,p,p,0] (mirrors llm_graph_input_pos::set_input); standard rope = [p]. |
| for (size_t k = 0; k < inp_pos_steps.size(); ++k) { |
| @@ -570,7 +570,7 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { |
| void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { |
| // opencoti F5 M6-S4 mtp: guard EACH setter on its own tensor's backend buffer, independently β |
| // and in particular DECOUPLE the kq_mask setter from the self_k_idxs guard. The read-only MTP |
| - // draft graph (build_attn_mtp) reuses this iswa input for a cross-read of the target KV but |
| + // dual-ctx assistant draft graph shares the target KV through mem_other, but |
| // writes no KV, so galloc prunes the unused self_*_idxs / self_*_rot write tensors (null buffer) |
| // while KEEPING the kq_mask it actually reads. Stock 0.10.3 nests set_input_kq_mask under the |
| // `self_k_idxs && self_k_idxs->buffer` guard, which would skip the LIVE mask exactly in the MTP |
| @@ -2999,95 +2999,9 @@ ggml_tensor * llm_graph_context::build_attn( |
| return cur; |
| } |
| |
| -// opencoti F5 M6-S4 mtp |
| -ggml_tensor * llm_graph_context::build_attn_mtp( |
| - llm_graph_input_attn_kv_iswa * inp, |
| - ggml_tensor * wo, |
| - ggml_tensor * wo_b, |
| - ggml_tensor * q_cur, |
| - ggml_tensor * kq_b, |
| - ggml_tensor * sinks, |
| - ggml_tensor * v_mla, |
| - float kq_scale, |
| - int il_mtp, |
| - int32_t il_kv_tgt, |
| - bool read_from_swa_kv, |
| - int64_t kv_embd_head_v, |
| - int64_t kv_n_head_v, |
| - bool use_k_as_v) const { |
| - auto * k_rot = read_from_swa_kv ? inp->self_k_rot_swa : inp->self_k_rot; |
| - auto * v_rot = read_from_swa_kv ? inp->self_v_rot_swa : inp->self_v_rot; |
| - |
| - if (k_rot) { |
| - q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); |
| - } |
| - if (v_rot) { |
| - // V-only rotation is applied after MHA in the standard path; no V write here. |
| - } |
| - |
| - ggml_build_forward_expand(gf, q_cur); |
| - |
| - const auto * mctx_iswa = inp->mctx; |
| - const auto * mctx_cur = read_from_swa_kv ? mctx_iswa->get_swa() : mctx_iswa->get_base(); |
| - |
| - const auto & kq_mask = read_from_swa_kv ? inp->get_kq_mask_swa() : inp->get_kq_mask(); |
| - |
| - ggml_tensor * q = q_cur; |
| - ggml_tensor * k = mctx_cur->get_k(ctx0, il_kv_tgt); |
| - ggml_tensor * v = use_k_as_v ? k : mctx_cur->get_v(ctx0, il_kv_tgt); |
| - |
| - if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0 |
| - || k->type == GGML_TYPE_TURBO3_TCQ || k->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) |
| - // opencoti F5 M6-S4 mtp (P3 R1 fix): do NOT pre-rotate Q here. build_attn_mha owns the |
| - // authoritative fused-turbo path β it forward-rotates Q (ggml_turbo_wht dir=0, graph.cpp |
| - // :1950) AND applies the paired output inverse-WHT (dir=1, :2001) for turbo2/3/4 K==V at |
| - // head_dimβ{128,256}, decode n_q. The MTP cross-read (use_k_as_v, k->type==v->type, hd 256 |
| - // SWA) triggers that exact branch, so a pre-rotation here WHT'd Q twice and left an unpaired |
| - // inverse on the output β wrong logits (graph still *built*, which is why P2's inert build |
| - // passed; the error only surfaces once P3 reads the draft). We keep only the pad/contiguity |
| - // prep build_attn_mha's ggml_turbo_wht contiguity assert relies on. |
| - if (q->ne[0] % 128 != 0) { |
| - const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0]; |
| - q = ggml_pad(ctx0, q, pad, 0, 0, 0); |
| - } |
| - if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); } |
| - } |
| - |
| - ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il_mtp); |
| - cb(cur, "kqv_out_mtp", il_mtp); |
| - |
| - if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0 |
| - || v->type == GGML_TYPE_TURBO3_TCQ || v->type == GGML_TYPE_TURBO2_TCQ) { // opencoti-hook: TCQ (#447/#500) |
| - const int64_t orig_v_head = kv_embd_head_v; |
| - const int64_t padded_v_head = v->ne[0]; |
| - if (padded_v_head != orig_v_head) { |
| - const int64_t n_head_v = kv_n_head_v; |
| - const int64_t n_tokens_cur = cur->ne[1]; |
| - cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur); |
| - cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur, |
| - cur->nb[1], cur->nb[2], 0); |
| - cur = ggml_cont(ctx0, cur); |
| - cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur); |
| - } |
| - } |
| - |
| - if (v_rot) { |
| - cur = ggml_mul_mat_aux(ctx0, cur, v_rot); |
| - } |
| - |
| - if (wo) { |
| - cur = build_lora_mm(wo, cur); |
| - cb(cur, "mtp_wo_out", il_mtp); |
| - } |
| - if (wo_b) { |
| - cur = ggml_add(ctx0, cur, wo_b); |
| - } |
| - |
| - return cur; |
| -} |
| |
| // opencoti fused-NextN (Option A / Gemma-mirror): read-only cross-attention into the frozen prefix |
| -// KV on the UNIFIED cache. Twin of build_attn_mtp for the non-iSWA llm_graph_input_attn_kv used by |
| +// KV on the UNIFIED cache via the non-iSWA llm_graph_input_attn_kv used by |
| // Qwen NextN drafters. Reads mctx->get_k/get_v(il) with the input's kq_mask β which exposes the |
| // full prefix 0..pmax because get_n_kv reports the cache's used extent, not the mtp_slot_info cell |
| // count. Writes NO draft KV; recurrence is carried by the hidden-state chain. wo is applied by the |
| |
| |
| |
| |
| @@ -595,7 +595,7 @@ struct llm_graph_params { |
| // opencoti F5 M6-S4 mtp (P4 fused): number of MTP draft steps to unroll into ONE graph. |
| // 1 = single-step (the proven P3 path). >1 chains each step's on-device argmax + backbone |
| // into the next step with no host round-trip / per-step GPU sync (the throughput fix). |
| - // Read by llm_build_gemma4_mtp; default 1 keeps every non-MTP construction unchanged. |
| + // Read by the fused NextN builders; default 1 keeps every non-MTP construction unchanged. |
| int32_t n_mtp_steps = 1; |
| |
| // return true if the "other" params would result in a graph with the same topology as with the current params |
| @@ -1021,28 +1021,8 @@ struct llm_graph_context { |
| int il, |
| bool cpu_fa_tail = false) const; // S3-b2 (#353): FA the host tail on the CPU (no DMA) |
| |
| - // opencoti F5 M6-S4 mtp |
| - // Gemma 4 MTP: cross-read target KV at il_kv_tgt from SWA or base cache; no KV write (k_cur/v_cur absent). |
| - // kv_* describe the **target** cache tensor layout at il_kv_tgt (for turbo V unpadded head extract). |
| - // When use_k_as_v is true, V tensor is replaced by K (HF Gemma4 assistant full-layer shortcut). |
| - ggml_tensor * build_attn_mtp( |
| - llm_graph_input_attn_kv_iswa * inp, |
| - ggml_tensor * wo, |
| - ggml_tensor * wo_b, |
| - ggml_tensor * q_cur, |
| - ggml_tensor * kq_b, |
| - ggml_tensor * sinks, |
| - ggml_tensor * v_mla, |
| - float kq_scale, |
| - int il_mtp, |
| - int32_t il_kv_tgt, |
| - bool read_from_swa_kv, |
| - int64_t kv_embd_head_v, |
| - int64_t kv_n_head_v, |
| - bool use_k_as_v) const; |
| - |
| // opencoti fused-NextN (Option A): read-only cross-attention into the frozen prefix KV on the |
| - // UNIFIED cache. Twin of build_attn_mtp for llm_graph_input_attn_kv (non-iSWA). Reads |
| + // UNIFIED cache (llm_graph_input_attn_kv). Reads |
| // mctx->get_k/get_v(il) with the input's kq_mask (which exposes 0..pmax via get_n_kv's used |
| // extent), runs build_attn_mha, writes NO draft KV. wo is applied by the caller. |
| // See docs/features/fused_nextn_mtp.md. |
| |
| |
| |
| |
| @@ -247,21 +247,6 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_full() { |
| return std::make_unique<llama_kv_cache_iswa_context>(this); |
| } |
| |
| -// opencoti F5 M6-S4 mtp |
| -llama_memory_context_ptr llama_kv_cache_iswa::init_mtp(llama_seq_id seq_id, llama_ubatch ubatch) { |
| - llama_kv_cache::slot_info_vec_t sinfos_base; |
| - llama_kv_cache::slot_info_vec_t sinfos_swa; |
| - |
| - sinfos_base.push_back(kv_base->mtp_slot_info(seq_id)); |
| - sinfos_swa.push_back(kv_swa->mtp_slot_info(seq_id)); |
| - |
| - std::vector<llama_ubatch> ubatches; |
| - ubatches.push_back(std::move(ubatch)); |
| - |
| - return std::make_unique<llama_kv_cache_iswa_context>( |
| - this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches)); |
| -} |
| - |
| llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) { |
| return std::make_unique<llama_kv_cache_iswa_context>(this, lctx, optimize); |
| } |
| |
| |
| |
| |
| @@ -77,8 +77,6 @@ public: |
| |
| llama_memory_context_ptr init_full() override; |
| |
| - // opencoti F5 M6-S4 mtp: read-only cross-read context over existing target cells (no slot alloc). |
| - llama_memory_context_ptr init_mtp(llama_seq_id seq_id, llama_ubatch ubatch); |
| |
| llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; |
| |
| |
| |
| |
| |
| @@ -5439,7 +5439,7 @@ llama_kv_cache_context::llama_kv_cache_context( |
| llama_kv_cache::slot_info_vec_t sinfos, |
| std::vector<llama_ubatch> ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) { |
| // opencoti F5 M6-S4 mtp: pre-compute n_kv so read-only paths (the MTP draft forward, |
| - // which runs process_ubatch_mtp and never calls apply()) get a valid mask/K/V shape from |
| + // which builds its graph inline and never calls apply()) get a valid mask/K/V shape from |
| // current cache occupancy. Without this, n_kv stays uninitialized and get_k() builds a |
| // view with a garbage dim-2 (observed ne[2]=2956782432 β ggml.c view-bounds assert). For |
| // paths that DO call apply(), n_kv is re-derived there after apply_ubatch(), so this is inert. |
| |
| |
| |
| |
| @@ -4,380 +4,10 @@ |
| #include <cmath> |
| #include <vector> |
| |
| -static llm_graph_params graph_params_for_mtp(llm_graph_params p, const llama_model & mtp_model) { |
| - p.arch = mtp_model.arch; |
| - p.hparams = mtp_model.hparams; |
| - p.gtype = LLM_GRAPH_TYPE_MTP; |
| - return p; |
| -} |
| - |
| -// Last layer in [range_start, range_end) whose attention type matches want_swa. |
| -static int32_t gemma4_mtp_kv_layer_last_in_range( |
| - const llama_hparams & tgt, int32_t range_start, int32_t range_end, bool want_swa) { |
| - int32_t best = -1; |
| - if (range_start < 0) { |
| - range_start = 0; |
| - } |
| - if (range_end > (int32_t) tgt.n_layer) { |
| - range_end = (int32_t) tgt.n_layer; |
| - } |
| - for (int32_t il = range_start; il < range_end; ++il) { |
| - if (tgt.is_swa((uint32_t) il) == want_swa) { |
| - best = il; |
| - } |
| - } |
| - return best; |
| -} |
| - |
| -// Build a single MTP step: token embedding (from target) + h_prev -> N transformer blocks -> |
| -// (post-projected backbone hidden, vocabulary logits, argmax token id). |
| -// |
| -// Used by both the single-step path (n_mtp_steps==1) and the chained path |
| -// (n_mtp_steps>1, called n_mtp_steps times within the same graph). |
| -// |
| -// `pos_step` must be a scalar I32 tensor [1] holding the absolute target position |
| -// for this step's RoPE (the cross-attn mask is shared across steps because the |
| -// target's KV cache only contains positions β€ attn_pos and all step positions |
| -// are > attn_pos, so causal/SWA admit all target cells uniformly). |
| -// |
| -// `out_logits`: F32 [n_vocab, 1] (full row for ordered embeddings too β required for greedy match with verify). |
| -// `out_argmax` (I32 [1]): greedy token id on-device. |
| -static void gemma4_mtp_build_one_step( |
| - const llm_graph_context & gctx, |
| - const llama_model & target, |
| - const llama_model & mtp, |
| - llm_graph_input_attn_kv_iswa * inp_attn, |
| - ggml_tensor * tok_step, // I32 [1] |
| - ggml_tensor * h_step, // F32 [n_bb, 1] |
| - ggml_tensor * pos_step, // I32 [1] |
| - ggml_tensor ** out_logits, // F32 [n_vocab, 1] |
| - ggml_tensor ** out_h_post, // F32 [n_bb, 1] |
| - ggml_tensor ** out_argmax) { // I32 [1] |
| - auto * ctx0 = gctx.ctx0; |
| - auto * gf = gctx.gf; |
| - const auto & hparams = gctx.hparams; |
| - const auto & cparams = gctx.cparams; |
| - const int n_layer = (int) gctx.n_layer; |
| - const int n_tokens = (int) gctx.n_tokens; |
| - const int n_ctx_orig = (int) gctx.n_ctx_orig; |
| - const int rope_type = gctx.rope_type; |
| - const float ext_factor = gctx.ext_factor; |
| - const float attn_factor = gctx.attn_factor; |
| - const float beta_fast = gctx.beta_fast; |
| - const float beta_slow = gctx.beta_slow; |
| - auto cb = [&](ggml_tensor * t, const char * name, int il) { gctx.cb(t, name, il); }; |
| - |
| - const float tok_scale = sqrtf((float) target.hparams.n_embd); |
| - |
| - ggml_tensor * tok_e = ggml_get_rows(ctx0, target.tok_embd, tok_step); |
| - cb(tok_e, "mtp_tgt_tok_embd", -1); |
| - |
| - // Gemma 4 scales token embeddings by sqrt(n_embd) at the input pipeline (gemma4-iswa.cpp). |
| - // Use target n_embd so Edge / non-Edge targets match the main forward. |
| - tok_e = ggml_scale(ctx0, tok_e, tok_scale); |
| - cb(tok_e, "mtp_tgt_tok_embd_scaled", -1); |
| - |
| - ggml_tensor * inp_cat = ggml_concat(ctx0, tok_e, h_step, 0); |
| - cb(inp_cat, "mtp_concat", -1); |
| - |
| - ggml_tensor * inpL = gctx.build_lora_mm(mtp.mtp_pre_projection, inp_cat); |
| - cb(inpL, "mtp_pre_proj_out", -1); |
| - |
| - ggml_build_forward_expand(gf, inpL); |
| - |
| - ggml_tensor * cur = nullptr; |
| - |
| - for (int il = 0; il < n_layer; ++il) { |
| - const int64_t n_embd_head = hparams.n_embd_head_k(il); |
| - GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); |
| - |
| - const int64_t n_head = hparams.n_head(il); |
| - |
| - const float freq_base_l = mtp.get_rope_freq_base(cparams, il); |
| - const float freq_scale_l = mtp.get_rope_freq_scale(cparams, il); |
| - const int n_rot_l = hparams.n_rot(il); |
| - |
| - cur = gctx.build_norm(inpL, mtp.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); |
| - cb(cur, "attn_norm", il); |
| - |
| - ggml_tensor * freq_factors = nullptr; |
| - if (!hparams.is_swa(il)) { |
| - freq_factors = mtp.layers[il].rope_freqs; |
| - } |
| - |
| - ggml_tensor * Qcur = gctx.build_lora_mm(mtp.layers[il].wq, cur); |
| - cb(Qcur, "Qcur", il); |
| - |
| - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); |
| - |
| - Qcur = gctx.build_norm(Qcur, mtp.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); |
| - cb(Qcur, "Qcur_normed", il); |
| - |
| - Qcur = ggml_rope_ext(ctx0, Qcur, pos_step, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, |
| - ext_factor, attn_factor, beta_fast, beta_slow); |
| - cb(Qcur, "Qcur_pos", il); |
| - |
| - const bool read_swa = hparams.is_swa(il); |
| - |
| - const int32_t n_tgt = (int32_t) target.hparams.n_layer; |
| - |
| - // Per HF Gemma4AssistantForCausalLM (transformers main): MTP cross-attention reads |
| - // ONE shared KV per attention type from the target β the LAST layer of that type. |
| - // ref: src/transformers/models/gemma4_assistant/modeling_gemma4_assistant.py |
| - // shared_kv_states = {"full_attention": (K, V), "sliding_attention": (K, V)} |
| - const int32_t il_kv = gemma4_mtp_kv_layer_last_in_range(target.hparams, 0, n_tgt, read_swa); |
| - |
| - GGML_ASSERT(il_kv >= 0 && "Gemma4 MTP: target has no layer matching MTP attention type (SWA/full)"); |
| - |
| - // Per HF Gemma4: even when target's attention_k_eq_v is True (so v_proj is None and |
| - // Vcur is created from Kcur source), the V cache slot is still WRITTEN with the |
| - // rms-norm-without-scale, non-rotated tensor. Therefore for cross-attention we must |
| - // ALWAYS fetch V from the cache β not reuse the post-RoPE K tensor. |
| - const bool use_k_as_v = false; |
| - |
| - const int64_t kv_embd_head_v = target.hparams.n_embd_head_v(il_kv); |
| - const int64_t kv_n_head_v = target.hparams.n_head_kv(il_kv); |
| - |
| - cur = gctx.build_attn_mtp(inp_attn, mtp.layers[il].wo, nullptr, Qcur, nullptr, nullptr, nullptr, |
| - hparams.f_attention_scale, il, il_kv, read_swa, kv_embd_head_v, kv_n_head_v, use_k_as_v); |
| - |
| - cur = gctx.build_norm(cur, mtp.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); |
| - cb(cur, "attn_post_norm", il); |
| - |
| - ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); |
| - cb(attn_out, "attn_out", il); |
| - |
| - GGML_ASSERT(mtp.layers[il].ffn_gate_inp == nullptr && "gemma4_assistant MTP does not support MoE FFN"); |
| - |
| - cur = gctx.build_norm(attn_out, mtp.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); |
| - cb(cur, "ffn_norm", il); |
| - |
| - cur = gctx.build_ffn(cur, |
| - mtp.layers[il].ffn_up, nullptr, nullptr, |
| - mtp.layers[il].ffn_gate, nullptr, nullptr, |
| - mtp.layers[il].ffn_down, nullptr, nullptr, |
| - nullptr, |
| - LLM_FFN_GELU, LLM_FFN_PAR, il); |
| - cb(cur, "ffn_out", il); |
| - |
| - cur = gctx.build_norm(cur, mtp.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); |
| - cb(cur, "ffn_post_norm", il); |
| - |
| - cur = ggml_add(ctx0, cur, attn_out); |
| - |
| - if (mtp.layers[il].out_scale) { |
| - cur = ggml_mul(ctx0, cur, mtp.layers[il].out_scale); |
| - cb(cur, "out_scaled", il); |
| - } |
| - |
| - cur = gctx.build_cvec(cur, il); |
| - cb(cur, "l_out", il); |
| - |
| - inpL = cur; |
| - } |
| - |
| - cur = inpL; |
| - |
| - cur = gctx.build_norm(cur, mtp.output_norm, nullptr, LLM_NORM_RMS, -1); |
| - cb(cur, "result_norm", -1); |
| - |
| - ggml_tensor * h_inner = cur; |
| - |
| - ggml_tensor * backbone = gctx.build_lora_mm(mtp.mtp_post_projection, h_inner); |
| - cb(backbone, "mtp_post_proj_out", -1); |
| - |
| - const int64_t n_vocab = mtp.tok_embd->ne[1]; |
| - const int64_t n_tokens_mtp = h_inner->ne[1]; |
| - |
| - if (mtp.hparams.use_ordered_embeddings) { |
| - // Centroid-routed LM head (HF Gemma4AssistantMaskedEmbedder): scatter candidate logits into a full |
| - // [n_vocab] row then argmax β matches masked full-vocab greedy (sparse-only argmax broke server accept). |
| - GGML_ASSERT(mtp.mtp_centroids != nullptr && mtp.mtp_token_ordering != nullptr); |
| - GGML_ASSERT(n_tokens_mtp == 1 && "ordered embeddings MTP expects a single token column"); |
| - const uint32_t n_c = mtp.hparams.n_centroids; |
| - const uint32_t top_k = mtp.hparams.centroid_top_k; |
| - GGML_ASSERT(n_c > 0 && top_k > 0 && (int64_t) top_k <= (int64_t) n_c); |
| - GGML_ASSERT(n_vocab % (int64_t) n_c == 0); |
| - const int64_t vsc = n_vocab / (int64_t) n_c; |
| - |
| - ggml_tensor * centroid_logits = gctx.build_lora_mm(mtp.mtp_centroids, h_inner); |
| - cb(centroid_logits, "mtp_centroid_logits", -1); |
| - |
| - ggml_tensor * topk_idx = ggml_top_k(ctx0, centroid_logits, (int) top_k); |
| - cb(topk_idx, "mtp_centroid_topk_idx", -1); |
| - |
| - const size_t ordering_nb1 = ggml_row_size(GGML_TYPE_I32, vsc); |
| - ggml_tensor * ordering = ggml_view_2d( |
| - ctx0, mtp.mtp_token_ordering, vsc, (int64_t) n_c, ordering_nb1, 0); |
| - cb(ordering, "mtp_token_ordering_view", -1); |
| - |
| - ggml_tensor * sel_ids = ggml_get_rows(ctx0, ordering, topk_idx); |
| - cb(sel_ids, "mtp_selected_token_ids", -1); |
| - |
| - const int64_t n_sel = (int64_t) top_k * vsc * n_tokens_mtp; |
| - ggml_tensor * flat_ids = ggml_reshape_1d(ctx0, sel_ids, n_sel); |
| - cb(flat_ids, "mtp_selected_token_ids_flat", -1); |
| - |
| - ggml_tensor * sel_emb = ggml_get_rows(ctx0, mtp.tok_embd, flat_ids); |
| - cb(sel_emb, "mtp_selected_embd", -1); |
| - |
| - ggml_tensor * sel_logits = gctx.build_lora_mm(sel_emb, h_inner); |
| - cb(sel_logits, "mtp_selected_logits", -1); |
| - ggml_tensor * sel_logits_f32 = ggml_cast(ctx0, sel_logits, GGML_TYPE_F32); |
| - |
| - ggml_tensor * logits_full = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_vocab, n_tokens_mtp); |
| - logits_full = ggml_fill_inplace(ctx0, logits_full, -1e30f); |
| - cb(logits_full, "mtp_logits_masked_base", -1); |
| - |
| - ggml_tensor * scatter_dst = ggml_cont_2d(ctx0, logits_full, 1, n_vocab * n_tokens_mtp); |
| - ggml_tensor * scatter_src = ggml_cont_2d(ctx0, sel_logits_f32, 1, n_sel); |
| - cur = ggml_set_rows(ctx0, scatter_dst, scatter_src, flat_ids); |
| - cb(cur, "mtp_logits_scatter_view", -1); |
| - cur = ggml_reshape_2d(ctx0, cur, n_vocab, n_tokens_mtp); |
| - cb(cur, "mtp_logits_full", -1); |
| - } else { |
| - cur = gctx.build_lora_mm(mtp.tok_embd, h_inner); |
| - cb(cur, "result_output_dense", -1); |
| - } |
| - |
| - if (hparams.f_final_logit_softcapping) { |
| - cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); |
| - cur = ggml_tanh(ctx0, cur); |
| - cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); |
| - } |
| - |
| - cb(cur, "result_output", -1); |
| - |
| - // Greedy argmax on-device: I32 [1] token index into the vocabulary row. |
| - ggml_tensor * arg = ggml_argmax(ctx0, cur); |
| - cb(arg, "result_argmax", -1); |
| - |
| - *out_logits = cur; |
| - *out_h_post = backbone; |
| - *out_argmax = arg; |
| -} |
| - |
| -llm_build_gemma4_mtp::llm_build_gemma4_mtp( |
| - const llama_model & target_model, |
| - const llama_model & mtp_model, |
| - const llm_graph_params & params) : |
| - llm_graph_context(graph_params_for_mtp(params, mtp_model)), |
| - target(target_model), |
| - mtp(mtp_model) { |
| - const int64_t n_bb = mtp.hparams.n_embd_out_impl; |
| - GGML_ASSERT(n_bb > 0); |
| - GGML_ASSERT(mtp.mtp_pre_projection != nullptr && mtp.mtp_post_projection != nullptr); |
| - |
| - // Step-0 inputs: the last accepted target token + its backbone hidden state. Steps 1..N-1 |
| - // (fused path) chain from the previous step's on-device argmax + post-projection, so only |
| - // step 0 reads token/h from the host. |
| - ggml_tensor * inp_tok = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); |
| - ggml_set_input(inp_tok); |
| - cb(inp_tok, "mtp_inp_last_token", -1); |
| - |
| - ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_bb, 1); |
| - ggml_set_input(inp_h); |
| - cb(inp_h, "mtp_inp_h_prev", -1); |
| - |
| - // opencoti F5 M6-S4 mtp (P4 fused): n_steps>1 unrolls the autoregressive draft chain into |
| - // ONE graph (no per-step host round-trip / GPU sync β the throughput fix). n_steps==1 is the |
| - // proven P3 single-step build, kept byte-for-byte (the logit-equiv harness + A/B fallback). |
| - const int32_t n_steps = std::max(params.n_mtp_steps, 1); |
| - |
| - auto * inp_attn = build_attn_inp_kv_iswa(); |
| - |
| - if (n_steps <= 1) { |
| - { |
| - auto inp_wrap = std::make_unique<llm_graph_input_mtp>(); |
| - inp_wrap->inp_last_token = inp_tok; |
| - inp_wrap->inp_h_prev = inp_h; |
| - res->add_input(std::move(inp_wrap)); |
| - } |
| - |
| - ggml_tensor * inp_pos = build_inp_pos(); |
| - |
| - ggml_tensor * logits = nullptr; |
| - ggml_tensor * h_post = nullptr; |
| - ggml_tensor * arg = nullptr; |
| - gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, |
| - inp_tok, inp_h, inp_pos, &logits, &h_post, &arg); |
| - |
| - res->t_embd = h_post; |
| - res->t_logits = logits; |
| - res->t_argmax = arg; |
| - |
| - ggml_build_forward_expand(gf, arg); |
| - ggml_build_forward_expand(gf, h_post); |
| - return; |
| - } |
| - |
| - // Fused N-step draft: one bespoke I32[1] position input per step (the cross-attn mask is |
| - // shared across steps β see gemma4_mtp_build_one_step's contract). The chain ties step k+1's |
| - // token to step k's argmax and step k+1's h to step k's backbone, entirely on-device. |
| - auto inp_wrap = std::make_unique<llm_graph_input_mtp>(); |
| - inp_wrap->inp_last_token = inp_tok; |
| - inp_wrap->inp_h_prev = inp_h; |
| - inp_wrap->inp_pos_steps.reserve(n_steps); |
| - |
| - std::vector<ggml_tensor *> pos_steps; |
| - pos_steps.reserve(n_steps); |
| - for (int32_t k = 0; k < n_steps; ++k) { |
| - ggml_tensor * p = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); |
| - ggml_set_input(p); |
| - cb(p, "mtp_inp_pos_step", k); |
| - pos_steps.push_back(p); |
| - inp_wrap->inp_pos_steps.push_back(p); |
| - } |
| - res->add_input(std::move(inp_wrap)); |
| - |
| - ggml_tensor * tok_k = inp_tok; |
| - ggml_tensor * h_k = inp_h; |
| - |
| - std::vector<ggml_tensor *> step_args; |
| - step_args.reserve(n_steps); |
| - ggml_tensor * last_logits = nullptr; |
| - ggml_tensor * last_h_post = nullptr; |
| - |
| - for (int32_t k = 0; k < n_steps; ++k) { |
| - ggml_tensor * logits_k = nullptr; |
| - ggml_tensor * h_post_k = nullptr; |
| - ggml_tensor * arg_k = nullptr; |
| - gemma4_mtp_build_one_step(*this, target, mtp, inp_attn, |
| - tok_k, h_k, pos_steps[k], &logits_k, &h_post_k, &arg_k); |
| - |
| - step_args.push_back(arg_k); |
| - last_logits = logits_k; |
| - last_h_post = h_post_k; |
| - |
| - // Chain on-device: next step's token = this step's greedy argmax (I32[1] index into the |
| - // target vocab β already used as a get_rows index inside build_one_step), next step's |
| - // hidden = this step's post-projected backbone (F32[n_bb,1]). No host round-trip. |
| - tok_k = arg_k; |
| - h_k = h_post_k; |
| - } |
| - |
| - // Collect the N per-step argmaxes into one I32[N] row for a single D2H read. Concat of I32 |
| - // is CUDA-unsupported (F32-only, by #253) so the scheduler runs it on the CPU backend β a |
| - // negligible side-consumer that does NOT touch the on-device get_rows chain above. |
| - ggml_tensor * all_args = step_args[0]; |
| - for (int32_t k = 1; k < n_steps; ++k) { |
| - all_args = ggml_concat(ctx0, all_args, step_args[k], 0); |
| - } |
| - cb(all_args, "mtp_fused_argmax", -1); |
| - |
| - res->t_argmax = all_args; // I32 [n_steps] β the drafted token block |
| - res->t_embd = last_h_post; // final backbone hidden -> next decode's h_prev seed |
| - res->t_logits = last_logits; // unused on the live path (out_logits == NULL) |
| - |
| - ggml_build_forward_expand(gf, all_args); |
| - ggml_build_forward_expand(gf, last_h_post); |
| -} |
| - |
| // opencoti F5 M6-S4 mtp: the GEMMA4_ASSISTANT sub-model (loaded into a gemma4 target via |
| // llama_model_load_mtp_from_file). Loading flows through the standard llama_model_load path |
| -// (llama_model_mapping -> this class -> load_arch_hparams/tensors). build_arch_graph throws: the |
| -// assistant is never a primary model; its graph (llm_build_gemma4_mtp) is built by the gemma4 |
| -// target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP. |
| +// (llama_model_mapping -> this class -> load_arch_hparams/tensors). The assistant is never a |
| +// primary model; build_arch_graph builds the dual-context drafter graph (ctx_other required). |
| void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { |
| hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; |
| ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); |
| @@ -443,7 +73,7 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { |
| // |
| // opencoti bug-858 (dual-context MTP): pre_projection is classified LLM_TENSOR_LAYER_INPUT, so the |
| // plain create_tensor lands it on the CPU buft. The single-context facade tolerates that (it runs on |
| - // the target's sched_mtp, which has the CPU backend registered), but a REAL draft context ctx_dft |
| + // the target's scheduler, which has the CPU backend registered), but a REAL draft context ctx_dft |
| // (llama_init_from_model over the nested assistant) registers only the assistant's *offloaded* (CUDA) |
| // weight buffers β a CPU-resident weight then reads back uninitialized β the whole drafter forward is |
| // NaN β 0% accept. Force it onto the output-head (GPU-when-offloaded) buft, exactly like tok_embd |
| @@ -502,14 +132,12 @@ void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { |
| // wk/wv): each attention layer reads the TARGET's shared K/V *in place* via the standard iSWA |
| // build_attn (Qcur only, null k_cur/v_cur -> no store). The draft context's create_memory (A3) |
| // shares the target's KV cells and aliases the target's last full (n_layer-1) / SWA (n_layer-2) |
| -// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V β |
| -// which is exactly what the single-context path did explicitly via build_attn_mtp(..., il_kv). |
| +// layer per draft layer, so build_attn at layer il transparently reads the aliased target K/V. |
| // The input token embedding + backbone hidden come from the target model via ctx_other. |
| +// (The retired single-context facade did this explicitly via a build_attn_mtp cross-read; removed |
| +// in #611 once this dual-context path proved parity β see UPSTREAM_SYNC.md.) |
| // |
| -// This is DISTINCT from the single-context llm_build_gemma4_mtp (built by the gemma4 TARGET's |
| -// build_arch_graph, gemma4.cpp) which is kept fully intact as the fallback drafter. |
| -// |
| -// Faithful to upstream b9859 (and unlike our single-context build): dense LM head over tok_embd |
| +// Faithful to upstream b9859: dense LM head over tok_embd |
| // (no ordered-embeddings/centroid path β b9859 loads centroids but ignores them in the graph), |
| // no final-logit softcapping (monotonic -> argmax-invariant), and no control-vector application |
| // (inert without a control vector). See docs/evaluations/mtp.md. |
| @@ -655,8 +283,6 @@ struct llm_build_gemma4_assistant_dual : public llm_graph_context { |
| std::unique_ptr<llm_graph_context> llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { |
| // opencoti bug-858 dual-context MTP (A6): build the gemma4-assistant drafter graph. This model |
| // is KV-less and only valid as an MTP draft context (ctx_dft) created FROM it with |
| - // cparams.ctx_other = the Gemma 4 target β the ctor GGML_ASSERTs that. The single-context |
| - // engine (llm_build_gemma4_mtp) is built by the gemma4 TARGET's build_arch_graph instead and |
| - // never reaches here; it remains the fallback drafter. |
| + // cparams.ctx_other = the Gemma 4 target β the ctor GGML_ASSERTs that. |
| return std::make_unique<llm_build_gemma4_assistant_dual>(*this, params); |
| } |
| |
| |
| |
| |
| @@ -132,14 +132,6 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) { |
| } |
| |
| std::unique_ptr<llm_graph_context> llama_model_gemma4::build_arch_graph(const llm_graph_params & params) const { |
| - // opencoti F5 M6-S4 mtp: when the spec framework requests an MTP graph (--spec-type draft-assistant), |
| - // build the Gemma-4 assistant drafter graph against the target's KV. Requires the assistant GGUF |
| - // loaded via llama_model_load_mtp_from_file. The llm_build_gemma4_mtp ctor re-derives arch/hparams/ |
| - // gtype from mtp_assistant (graph_params_for_mtp), so raw params are passed. |
| - if (params.gtype == LLM_GRAPH_TYPE_MTP) { |
| - GGML_ASSERT(mtp_assistant && "GEMMA4 MTP graph requires llama_model_load_mtp_from_file on the target model"); |
| - return std::make_unique<llm_build_gemma4_mtp>(*this, *mtp_assistant, params); |
| - } |
| return std::make_unique<graph>(*this, params); |
| } |
| |
| |
| |
| |
| |
| @@ -809,19 +809,10 @@ struct llama_model_gemma4 : public llama_model_base { |
| std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; |
| }; |
| |
| -// opencoti F5 M6-S4 mtp |
| -// Gemma 4 MTP: target model supplies tok_embd rows + KV; mtp_model supplies assistant weights. |
| -struct llm_build_gemma4_mtp : public llm_graph_context { |
| - const llama_model & target; |
| - const llama_model & mtp; |
| - |
| - llm_build_gemma4_mtp(const llama_model & target, const llama_model & mtp_model, const llm_graph_params & params); |
| -}; |
| - |
| // opencoti F5 M6-S4 mtp: Gemma 4 MTP assistant β loaded INTO a gemma4 target via |
| // llama_model_load_mtp_from_file (never as a primary -m model). load_arch_* read the assistant's own |
| -// hparams/tensors; build_arch_graph throws (the assistant graph is llm_build_gemma4_mtp, built by the |
| -// gemma4 target's build_arch_graph when gtype==LLM_GRAPH_TYPE_MTP). |
| +// hparams/tensors; build_arch_graph builds the dual-context drafter graph (requires |
| +// cparams.ctx_other = the Gemma 4 target, bug-858). |
| struct llama_model_gemma4_assistant : public llama_model_base { |
| llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} |
| void load_arch_hparams(llama_model_loader & ml) override; |
| |
| |
| |
| |
| @@ -628,7 +628,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr |
| // context's prior decodes; we do NOT write draft KV here β recurrence is carried by the |
| // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used |
| // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here β pruned. |
| - // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. |
| + // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. |
| cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); |
| } else { |
| Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, |
| @@ -700,8 +700,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr |
| return; |
| } |
| |
| - // opencoti-hook: fused-nextn-mtp β fused N-step draft chain (mirrors gemma4-assistant |
| - // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One |
| + // opencoti-hook: fused-nextn-mtp β fused N-step draft chain. DORMANT until the driver |
| + // sets n_mtp_steps>1 (S3). One |
| // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 |
| // ids/token β ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] |
| // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, |
| |
| |
| |
| |
| @@ -680,7 +680,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm |
| // context's prior decodes; we do NOT write draft KV here β recurrence is carried by the |
| // hidden-state chain. mtp_slot_info gives one cell (pmax); get_n_kv reports the full used |
| // extent so the causal kq_mask exposes 0..pmax. Kcur/Vcur above go unused here β pruned. |
| - // Mirrors build_attn_mtp (llama-graph.cpp). See docs/features/fused_nextn_mtp.md delta #6. |
| + // Read-only cross-attention into the frozen prefix. See docs/features/fused_nextn_mtp.md delta #6. |
| cur = build_attn_readonly_nextn(inp_attn, Qcur, kq_scale, il); |
| } else { |
| Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, |
| @@ -784,8 +784,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm |
| return; |
| } |
| |
| - // opencoti-hook: fused-nextn-mtp β fused N-step draft chain (mirrors gemma4-assistant |
| - // llm_build_gemma4_mtp). DORMANT until the driver sets n_mtp_steps>1 (S3). One |
| + // opencoti-hook: fused-nextn-mtp β fused N-step draft chain. DORMANT until the driver |
| + // sets n_mtp_steps>1 (S3). One |
| // I32[n_pos_per_embd] position input per step (bug-870: mrope archs like qwen35 need 4 |
| // ids/token β ggml_rope_multi asserts a->ne[2]*4==b->ne[0]; the setter fills the [p,p,p,0] |
| // M-RoPE text layout, or [p] for standard rope). Step k+1's token = step k's on-device argmax, |
| |
| |
| |
| |
| @@ -918,25 +918,21 @@ private: |
| |
| add_bos_token = llama_vocab_get_add_bos(vocab); |
| |
| - // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4_assistant draft head is a SEPARATE GGUF (so |
| - // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file β |
| - // no second llama_context / KV cache. Its cross-attention reads the target's KV read-only at |
| - // decode time and "draft decodes" run on ctx_tgt via llama_decode_mtp. This is distinct from |
| - // both native branches (separate-model draft below; NextN MTP-ctx-against-target further |
| - // down). Detect it via the enabled types vector, load into the target, and leave |
| - // model_dft / ctx_dft null so the generic draft-context paths are skipped (the framework's |
| - // common_speculative_init then validates the in-target assistant via the draft.ctx_tgt set here). |
| + // opencoti F5 M6-S4 mtp: the Gemma-4 gemma4-assistant draft head is a SEPARATE GGUF (so |
| + // has_dft() is true) but loads INTO the target model via llama_model_load_mtp_from_file, |
| + // then runs as a REAL draft context (ctx_dft) whose cparams.ctx_other = ctx_tgt shares the |
| + // target's KV (bug-858 dual-context, upstream b9859 shape). This is distinct from both |
| + // native branches (separate-model draft below; NextN MTP-ctx-against-target further down). |
| + // Detect it via the enabled types vector and load into the target first. |
| const bool spec_assistant = std::find(params_base.speculative.types.begin(), |
| params_base.speculative.types.end(), |
| COMMON_SPECULATIVE_TYPE_MTP) != params_base.speculative.types.end(); |
| if (params_base.speculative.has_dft() && spec_assistant) { |
| - // opencoti F5 M6-S4 mtp (#485): the single-context assistant engine cross-reads the target |
| - // KV in place (ctx_dft==nullptr, build_attn_mtp). With --parallel >1 WITHOUT --kv-unified the |
| - // KV is per-stream-split (patch 0031); get_k/get_v then return per-stream tensors whose layout |
| - // the MTP cross-read reshape does not handle -> GGML reshape assert at the first draft decode. |
| - // The unified cache IS handled: proven real_frac=0 + ~93% draft acceptance @ --parallel 2 |
| - // --kv-unified, which is exactly the PolyKV config (PolyKV forces --kv-unified). Require it |
| - // explicitly and fail clean at boot instead of crashing mid-request. |
| + // opencoti F5 M6-S4 mtp (#485/#611): with --parallel >1 WITHOUT --kv-unified the KV is |
| + // per-stream-split (patch 0031); the assistant's shared-KV aliasing (mem_other) assumes the |
| + // unified per-layer tensor layout, so per-stream splits are not supported. The unified |
| + // cache IS handled (proven on the facade at --parallel 2 --kv-unified β the PolyKV config, |
| + // which forces --kv-unified). Require it explicitly and fail clean at boot. |
| if (params_base.n_parallel > 1 && !params_base.kv_unified) { |
| SRV_ERR("%s", |
| "Gemma assistant-MTP (--spec-type draft-assistant) with --parallel >1 requires " |
| @@ -965,21 +961,19 @@ private: |
| return false; |
| } |
| |
| - // opencoti bug-858 dual-context MTP (A5): optionally create a REAL draft context FROM the |
| - // nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 |
| + // opencoti bug-858 dual-context MTP (A5, default since #611): create a REAL draft context |
| + // FROM the nested gemma4-assistant model with cparams.ctx_other = ctx_tgt (upstream b9859 |
| // dual-context). The assistant is KV-less: its create_memory (A3) shares the target's KV |
| // cells + aliases the last full/SWA layer, and its graph (A6) reads the target's tok_embd |
| // + shared K/V via ctx_other; the shared-KV draft_mtp driver (is_mem_shared) then runs it. |
| - // Gated by OPENCOTI_MTP_DUAL_CTX so the default keeps the proven single-context in-target |
| - // path (ctx_dft == nullptr, llama_decode_mtp) byte-identical. |
| - const char * mtp_dual_env = getenv("OPENCOTI_MTP_DUAL_CTX"); |
| - const bool mtp_dual_ctx = mtp_dual_env && mtp_dual_env[0] && mtp_dual_env[0] != '0'; |
| - if (mtp_dual_ctx) { |
| + // This is the ONLY assistant execution path β the single-context in-target facade |
| + // (llama_decode_mtp) was removed in #611 after the port proved accept parity. |
| + { |
| // llama_init_from_model wants a mutable model*; the nested assistant is owned mutably |
| // by the target (unique_ptr<llama_model>), so the const_cast is safe. |
| llama_model * assistant = const_cast<llama_model *>(llama_model_get_mtp_assistant(model_tgt)); |
| if (assistant == nullptr) { |
| - SRV_ERR("%s", "MTP dual-context requested but the assistant is not loaded into the target\n"); |
| + SRV_ERR("%s", "MTP assistant is not loaded into the target\n"); |
| return false; |
| } |
| |
| @@ -1002,11 +996,6 @@ private: |
| params_base.speculative.draft.ctx_tgt = ctx_tgt; |
| params_base.speculative.draft.ctx_dft = ctx_dft.get(); |
| SRV_INF("%s", "MTP assistant dual-context draft created (ctx_other=target, shared KV)\n"); |
| - } else { |
| - // No separate draft context: the assistant decodes on ctx_tgt via llama_decode_mtp. |
| - params_base.speculative.draft.ctx_tgt = ctx_tgt; |
| - params_base.speculative.draft.ctx_dft = nullptr; |
| - SRV_INF("%s", "MTP assistant loaded into target (single-context)\n"); |
| } |
| } else if (params_base.speculative.has_dft()) { |
| // TODO speculative: move to common/speculative.cpp? |
|
|