From: opencoti Subject: [PATCH 0072] F5 M6 PolyKV S1 — SharedKVPool shared read-only prefix Server-side SharedKVPool (F5 M6 S1, #370): a read-only KV prefix shared across sequences/slots so N agents over one base context pay O(1) shared-prefix memory instead of O(N). Per-request opt-in via the `shared_pool_slot` (>=0 selects the pool donor slot) and `shared_prefix_n_tokens` task params; default-off (slot -1) is byte-identical to upstream slot handling. Pure server/task plumbing — no ggml or kernel change. Bench: perf/llamafile/advanced-kv-stack.bench.ts (A4/S'); gate .opencoti/m6-s1-shared-pool-gate.sh Milestone: F5 M6 PolyKV S1 — see docs/features/poly_kv.md Applies after: 0071 (numbered 0072 so lexical apply order lands it after the M7 rolling-KV 0070/0071 it composes with; the original 0060 slot predates 0070). diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp --- a/llama.cpp/tools/server/server-context.cpp +++ b/llama.cpp/tools/server/server-context.cpp @@ -2907,6 +2907,48 @@ private: } } + // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md + // F5 M6 PolyKV S1: bit-share a pool slot's read-only prefix into this seq + // (zero-copy cells.seq_add in unified mode) and skip re-processing it. + // Caller contract: this request's prompt MUST begin with exactly the pool + // slot's first shared_prefix_n_tokens tokens, and the pool slot must remain + // resident while sharers reference it. RESIDENCY PREREQUISITE: the server + // must be booted with --no-clear-idle. The upstream idle-slot prompt-cache + // offload (clear_idle, default-on with --kv-unified + --cache-ram; PR #16391) + // serializes an idle slot's KV to a host blob and CLEARS its device cells the + // moment a new task launches — which would evict the pool prefix before any + // sharer's seq_cp runs (seq would be empty → silent miss). We guard against + // that below: if the pool's resident range does not cover [0,P), we skip the + // share and fall through to normal full re-processing (correct, just slower). + // We mirror the copy_state_to() precedent: after seq_cp we also set this + // slot's token bookkeeping to the first P tokens, else keep_first(n_past) + // asserts on the (empty) slot token list (server-common.cpp:397). + if (slot.task->params.shared_pool_slot >= 0 && + slot.task->params.shared_prefix_n_tokens > 0 && + slot.task->params.shared_pool_slot != slot.id) { + const int pool = slot.task->params.shared_pool_slot; + int32_t P = std::min(slot.task->params.shared_prefix_n_tokens, + (int32_t) input_tokens.size()); + auto * mem = llama_get_memory(ctx_tgt); + // residency guard: the pool seq must actually hold cells covering [0,P) + const llama_pos pool_max = llama_memory_seq_pos_max(mem, pool); + if (pool_max < (llama_pos) (P - 1)) { + SLT_WRN(slot, "poly-kv-pool: pool slot %d not resident to %d (max pos %d) — " + "skipping share, full reprocess (boot with --no-clear-idle)\n", + pool, (int) (P - 1), (int) pool_max); + } else { + // clear any stale KV on this slot, then bit-share the pool's [0,P) prefix + llama_memory_seq_rm(mem, slot.id, -1, -1); + llama_memory_seq_cp(mem, pool, slot.id, 0, P); + // mirror token bookkeeping so keep_first(n_past)/pos_next stay consistent + server_tokens shared = input_tokens.clone(); + shared.keep_first(P); + slot.prompt.tokens = std::move(shared); + n_past = P; + SLT_INF(slot, "poly-kv-pool: shared %d-token prefix from slot %d (n_past -> %d)\n", (int) P, pool, n_past); + } + } + // [TAG_PROMPT_LOGITS] if (n_past == slot.task->n_tokens() && n_past > 0) { SLT_WRN(slot, "need to evaluate at least 1 token for each active slot (n_past = %d, task.n_tokens() = %d)\n", n_past, slot.task->n_tokens()); diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp --- a/llama.cpp/tools/server/server-task.cpp +++ b/llama.cpp/tools/server/server-task.cpp @@ -273,6 +273,8 @@ task_params server_task::params_from_json_cmpl( params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1)); params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse); params.session_id = json_value(data, "session_id", std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + params.shared_pool_slot = json_value(data, "shared_pool_slot", -1); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md + params.shared_prefix_n_tokens = json_value(data, "shared_prefix_n_tokens", 0); // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); params.response_fields = json_value(data, "response_fields", std::vector()); diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h --- a/llama.cpp/tools/server/server-task.h +++ b/llama.cpp/tools/server/server-task.h @@ -72,6 +72,15 @@ struct task_params { // number of prompt tokens before the latest user message int32_t n_before_user = -1; + // opencoti-hook: poly-kv-pool — see docs/features/poly_kv.md + // F5 M6 PolyKV S1 (SharedKVPool). When shared_pool_slot >= 0, the server + // bit-shares this request's [0, shared_prefix_n_tokens) KV with the pool + // slot's already-resident read-only prefix (zero-copy cells.seq_add in + // unified mode), so N agents share ONE physical copy of the prefix and the + // prefix is neither re-stored nor re-processed. <0 (default) is fully inert. + int32_t shared_pool_slot = -1; + int32_t shared_prefix_n_tokens = 0; + int64_t t_max_prompt_ms = -1; // TODO: implement int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit