From: opencoti Subject: [PATCH 0006] kv-cache reuse for agentic turns — session→slot affinity opencoti routes an agentic loop through the bundled llamafile server: each turn N+1 re-sends turn N's context prefix (system prompt + tool defs + history) verbatim, then appends. The server already reuses KV for a matching prefix — cache_prompt defaults to true and get_available_slot() picks the idle slot whose cached token-prefix best matches the incoming prompt (above --slot-prompt-similarity, then LRU). That covers single-session turn-2 reuse at the default --parallel 1. What upstream lacks is *session affinity*. With --parallel N and several concurrent opencode sessions sharing slots, the LCP/LRU picker can route a session to a slot holding a different session's KV, evicting a prefix that would otherwise be reused. This patch adds an optional `session_id` request field that keys slot selection: - get_available_slot(): if the request carries a non-empty session_id that already maps to an idle slot, return that slot (its resident KV is this session's prefix by construction). The existing LCP→LRU path is the fallback. - launch_slot_with_task(): record session_id -> slot.id when a slot is committed to a session, dropping any stale session previously bound to that slot. The prefix match itself is still the server's existing token-LCP — affinity only ensures the right slot (the one holding the session's KV) is chosen, so "(session_id, prefix) reuse" falls out without a separate prefix hash. Regression shield: empty session_id (the default, and what every non-opencoti client sends) skips the affinity phase entirely, leaving slot selection byte-for-byte identical to upstream. Builds clean under cosmocc (std::map / std::string already available via server-task.h). Bench: perf/llamafile/turn-2-latency.bench.ts Milestone: F5 M0 — see docs/features/advanced_kv.md Design: docs/features/advanced_kv.md (M0) Upstreaming: deferred; candidate to propose upstream as a generic session-id slot-affinity hint once benched on opencoti workloads. --- diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp --- a/llama.cpp/tools/server/server-context.cpp +++ b/llama.cpp/tools/server/server-context.cpp @@ -698,6 +698,13 @@ private: // Necessary similarity of prompt for slot selection float slot_prompt_similarity = 0.0f; + // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + // session_id -> slot.id affinity, so an agentic session routes back to the + // slot holding its resident KV prefix (turn-to-turn reuse, and no + // cross-session eviction when --parallel > 1). Empty session_id is never + // inserted, so non-opencoti traffic never consults or mutates this map. + std::map session_to_slot; + std::string model_name; // name of the loaded model, to be used by API std::set model_aliases; // additional names for the model std::set model_tags; // informational tags @@ -1249,6 +1256,25 @@ private: bool update_cache = false; + // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + // Session affinity: if this session already owns an idle slot, reuse + // it. That slot's resident KV is the session's prefix, so the LCP path + // below would pick it anyway when free — but the explicit pin keeps a + // concurrent session from stealing/evicting it under --parallel > 1. + // Skipped entirely for empty session_id, so upstream selection (LCP + // then LRU, below) is unchanged for every non-opencoti request. + if (ret == nullptr && !task.params.session_id.empty()) { + auto it = session_to_slot.find(task.params.session_id); + if (it != session_to_slot.end()) { + for (server_slot & slot : slots) { + if (slot.id == it->second && !slot.is_processing()) { + ret = &slot; + break; + } + } + } + } + // find the slot that has at least n% prompt similarity if (ret == nullptr && slot_prompt_similarity != 0.0f) { float sim_best = 0; @@ -1498,6 +1524,22 @@ private: slot.smpl.reset(); } + // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + // Record session→slot affinity before `task` is moved into the slot, + // so this session's next turn routes back here and reuses its KV. A + // slot serves one session at a time, so first drop any stale session + // that previously mapped to this slot id. + if (!task.params.session_id.empty()) { + for (auto it = session_to_slot.begin(); it != session_to_slot.end(); ) { + if (it->second == slot.id && it->first != task.params.session_id) { + it = session_to_slot.erase(it); + } else { + ++it; + } + } + session_to_slot[task.params.session_id] = slot.id; + } + slot.task = std::make_unique(std::move(task)); slot.state = slot.task->is_child() diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp --- a/llama.cpp/tools/server/server-task.cpp +++ b/llama.cpp/tools/server/server-task.cpp @@ -272,6 +272,7 @@ task_params server_task::params_from_json_cmpl( params.n_discard = std::max(0, params.n_discard); params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1)); params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse); + params.session_id = json_value(data, "session_id", std::string("")); // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); params.response_fields = json_value(data, "response_fields", std::vector()); diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h --- a/llama.cpp/tools/server/server-task.h +++ b/llama.cpp/tools/server/server-task.h @@ -61,6 +61,14 @@ struct task_params { int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled) + // opencoti F5 M0 kv-reuse — see docs/features/advanced_kv.md + // Optional session affinity key. When non-empty, the server routes this + // request back to the slot the same session last used, reusing its + // resident KV prefix across agentic turns. Empty (the default, and what + // every non-opencoti client sends) preserves upstream slot selection + // byte-for-byte. + std::string session_id; + // number of prompt tokens before the latest user message int32_t n_before_user = -1;