From: opencoti Subject: [PATCH 0095] prefill checkpoint alignment: min-step 8192 + state-buffer recycle pool (bug-2105/2107, #614) - checkpoint_min_step 256 -> 8192 (align with b9859; stale port default caused several SWA/recurrent state checkpoints per request instead of <=1). - opencoti-hook: ckpt-buf-noinit — common_state_buf type + server-side recycle pool (ckpt_buf_pool, cap 4) so steady-state requests reuse warm pages: no GB-class value-init memset and no page-fault storm inside the state D2H copy. (Measured dead end: a non-zeroing allocator alone regressed all models 8-13%.) diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h index b688492..3bb6aa2 100644 --- a/llama.cpp/common/common.h +++ b/llama.cpp/common/common.h @@ -670,7 +673,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; @@ -1124,14 +1127,21 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); // prompt utils // +// opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — named buffer type for state snapshots so +// the server can recycle them across requests (warm pages: no memset, no page-fault storm). +// Measured dead end kept for the record: a non-zeroing allocator here made the COLD path +// slower (page faults land inside the state D2H copy loop instead of a fast sequential +// memset prefault) — plain vector cold + recycled warm is the optimum. +using common_state_buf = std::vector; + struct common_prompt_checkpoint { int64_t n_tokens; llama_pos pos_min; llama_pos pos_max; - std::vector data_tgt; - std::vector data_dft; + common_state_buf data_tgt; + common_state_buf data_dft; size_t size() const; diff --git a/llama.cpp/tools/server/server-context.cpp b/llama.cpp/tools/server/server-context.cpp index 7bad046..34b50b1 100644 --- a/llama.cpp/tools/server/server-context.cpp +++ b/llama.cpp/tools/server/server-context.cpp @@ -2119,20 +2158,50 @@ private: return true; } + // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — recycle retired checkpoint state + // buffers so the pages stay mapped and warm; a fresh GB-class allocation per request + // pays a page-fault storm inside the state D2H copy (worse than the memset it replaced) + std::vector ckpt_buf_pool; + + void ckpt_buf_recycle(common_prompt_checkpoint & ckpt) { + if (ckpt_buf_pool.size() < 4 && ckpt.data_tgt.capacity() > 0) { + ckpt_buf_pool.push_back(std::move(ckpt.data_tgt)); + } + if (ckpt_buf_pool.size() < 4 && ckpt.data_dft.capacity() > 0) { + ckpt_buf_pool.push_back(std::move(ckpt.data_dft)); + } + } + + common_state_buf ckpt_buf_acquire() { + if (ckpt_buf_pool.empty()) { + return {}; + } + common_state_buf buf = std::move(ckpt_buf_pool.back()); + ckpt_buf_pool.pop_back(); + return buf; + } + // n_tokens_cur: the number of tokens added to the batch for the current slot void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { // make room for the new checkpoint, if needed - const auto & cur = slot.prompt.checkpoints.front(); + auto & cur = slot.prompt.checkpoints.front(); SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); + ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); } auto & cur = slot.prompt.checkpoints.emplace_back(); + // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — seed with warm recycled buffers + cur.data_tgt = ckpt_buf_acquire(); + if (ctx_dft) { + cur.data_dft = ckpt_buf_acquire(); + } + cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); @@ -2980,9 +3042,10 @@ private: { // erase any checkpoints with pos_max > pos_next for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { - const auto & cur = *it; + auto & cur = *it; if (cur.pos_max > pos_next) { SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); + ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) it = slot.prompt.checkpoints.erase(it); } else { ++it; diff --git a/llama.cpp/tools/server/server-task.cpp b/llama.cpp/tools/server/server-task.cpp index 803e31f..70ca8dc 100644 --- a/llama.cpp/tools/server/server-task.cpp +++ b/llama.cpp/tools/server/server-task.cpp @@ -2025,8 +2025,8 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t } } - std::vector state_data_tgt; - std::vector state_data_dft; + common_state_buf state_data_tgt; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) + common_state_buf state_data_dft; // check if we can allocate enough memory for the new state try { diff --git a/llama.cpp/tools/server/server-task.h b/llama.cpp/tools/server/server-task.h index 933bd99..380dc27 100644 --- a/llama.cpp/tools/server/server-task.h +++ b/llama.cpp/tools/server/server-task.h @@ -582,8 +582,8 @@ struct server_task_result_apply_lora : server_task_result { }; struct server_prompt_data { - std::vector main; - std::vector drft; + common_state_buf main; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) + common_state_buf drft; size_t size() const { return main.size() + drft.size();