| From: opencoti |
| Subject: [PATCH 0095] prefill checkpoint alignment: min-step 8192 + state-buffer recycle pool (bug-2105/2107, #614) |
|
|
| - checkpoint_min_step 256 -> 8192 (align with b9859; stale port default caused |
| several SWA/recurrent state checkpoints per request instead of <=1). |
| - opencoti-hook: ckpt-buf-noinit — common_state_buf type + server-side recycle |
| pool (ckpt_buf_pool, cap 4) so steady-state requests reuse warm pages: no |
| GB-class value-init memset and no page-fault storm inside the state D2H copy. |
| (Measured dead end: a non-zeroing allocator alone regressed all models 8-13%.) |
|
|
| |
| |
| |
| |
| @@ -670,7 +673,7 @@ struct common_params { |
| bool cache_prompt = true; // whether to enable prompt caching |
| bool cache_idle_slots = true; // save and clear idle slots upon starting a new task |
| int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot |
| - int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints |
| + int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints |
| int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. |
| |
| std::string hostname = "127.0.0.1"; |
| @@ -1124,14 +1127,21 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *); |
| // prompt utils |
| // |
| |
| +// opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — named buffer type for state snapshots so |
| +// the server can recycle them across requests (warm pages: no memset, no page-fault storm). |
| +// Measured dead end kept for the record: a non-zeroing allocator here made the COLD path |
| +// slower (page faults land inside the state D2H copy loop instead of a fast sequential |
| +// memset prefault) — plain vector cold + recycled warm is the optimum. |
| +using common_state_buf = std::vector<uint8_t>; |
| + |
| struct common_prompt_checkpoint { |
| int64_t n_tokens; |
| |
| llama_pos pos_min; |
| llama_pos pos_max; |
| |
| - std::vector<uint8_t> data_tgt; |
| - std::vector<uint8_t> data_dft; |
| + common_state_buf data_tgt; |
| + common_state_buf data_dft; |
| |
| size_t size() const; |
| |
| |
| |
| |
| |
| @@ -2119,20 +2158,50 @@ private: |
| return true; |
| } |
| |
| + // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — recycle retired checkpoint state |
| + // buffers so the pages stay mapped and warm; a fresh GB-class allocation per request |
| + // pays a page-fault storm inside the state D2H copy (worse than the memset it replaced) |
| + std::vector<common_state_buf> ckpt_buf_pool; |
| + |
| + void ckpt_buf_recycle(common_prompt_checkpoint & ckpt) { |
| + if (ckpt_buf_pool.size() < 4 && ckpt.data_tgt.capacity() > 0) { |
| + ckpt_buf_pool.push_back(std::move(ckpt.data_tgt)); |
| + } |
| + if (ckpt_buf_pool.size() < 4 && ckpt.data_dft.capacity() > 0) { |
| + ckpt_buf_pool.push_back(std::move(ckpt.data_dft)); |
| + } |
| + } |
| + |
| + common_state_buf ckpt_buf_acquire() { |
| + if (ckpt_buf_pool.empty()) { |
| + return {}; |
| + } |
| + common_state_buf buf = std::move(ckpt_buf_pool.back()); |
| + ckpt_buf_pool.pop_back(); |
| + return buf; |
| + } |
| + |
| // n_tokens_cur: the number of tokens added to the batch for the current slot |
| void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { |
| while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { |
| // make room for the new checkpoint, if needed |
| - const auto & cur = slot.prompt.checkpoints.front(); |
| + auto & cur = slot.prompt.checkpoints.front(); |
| |
| SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", |
| cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); |
| |
| + ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) |
| slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); |
| } |
| |
| auto & cur = slot.prompt.checkpoints.emplace_back(); |
| |
| + // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) — seed with warm recycled buffers |
| + cur.data_tgt = ckpt_buf_acquire(); |
| + if (ctx_dft) { |
| + cur.data_dft = ckpt_buf_acquire(); |
| + } |
| + |
| cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); |
| |
| cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); |
| @@ -2980,9 +3042,10 @@ private: |
| { |
| // erase any checkpoints with pos_max > pos_next |
| for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) { |
| - const auto & cur = *it; |
| + auto & cur = *it; |
| if (cur.pos_max > pos_next) { |
| SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024); |
| + ckpt_buf_recycle(cur); // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) |
| it = slot.prompt.checkpoints.erase(it); |
| } else { |
| ++it; |
| |
| |
| |
| |
| @@ -2025,8 +2025,8 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t |
| } |
| } |
| |
| - std::vector<uint8_t> state_data_tgt; |
| - std::vector<uint8_t> state_data_dft; |
| + common_state_buf state_data_tgt; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) |
| + common_state_buf state_data_dft; |
| |
| // check if we can allocate enough memory for the new state |
| try { |
| |
| |
| |
| |
| @@ -582,8 +582,8 @@ struct server_task_result_apply_lora : server_task_result { |
| }; |
| |
| struct server_prompt_data { |
| - std::vector<uint8_t> main; |
| - std::vector<uint8_t> drft; |
| + common_state_buf main; // opencoti-hook: ckpt-buf-noinit (#614/bug-2107) |
| + common_state_buf drft; |
| |
| size_t size() const { |
| return main.size() + drft.size(); |
|
|
