From: opencoti Subject: [PATCH 0089] rolling-KV two-pass compute-buffer reserve (bug-1342) The POSITION_WINDOW resident-window sizer (opencoti_compute_resident_window_cells) held back a FIXED OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES = 1536 MiB for the compute scratch that graph_reserve allocates AFTER the KV cache. But that compute buffer SCALES with context — measured 3.63 GiB (3 800 045 568 B) at 262 144-524 288 ctx on an RTX PRO 6000 — so the sizer kept ~2.1 GiB too much KV resident and graph_reserve OOM'd at boot: ggml_gallocr_reserve_n_impl: failed to allocate CUDA0 buffer of size 3800045568 graph_reserve: failed to allocate compute buffers Fix: measure the real compute buffer instead of guessing it. Two-pass context init, gated to the GPU flash-attn residency-managed path so the common resident case stays single-pass / byte-identical: Pass 1 (measure): create_memory with kv_window_measure_pass=true -> the window sizer returns a MINIMAL resident window (tiny device KV, boots for ANY compute size). sched_reserve() runs the pp worst-case reserve; sum the GPU backends' compute buffer via backend_buf_exp_size. Pass 2 (real): re-create_memory with kv_compute_reserve_mib = measured + 256 MiB safety; the sizer now holds back the true compute buffer and sizes the window against (budget - model - measured). sched_need_reserve=true; sched_reserve() again. The compute buffer is independent of the window/tail split (it depends on n_ubatch x n_kv x n_embd, and n_kv = full ctx regardless of the resident split), so one measurement suffices. kv_compute_reserve_mib / kv_window_measure_pass are INTERNAL cparams (set by the context two-pass, not from llama_context_params); hybrid-memory paths pass 0/false (two-pass disabled). Host-only change — see docs/features/rolling_kv_compute_reserve.md. diff --git a/llama.cpp/src/llama-cparams.h b/llama.cpp/src/llama-cparams.h --- a/llama.cpp/src/llama-cparams.h +++ b/llama.cpp/src/llama-cparams.h @@ -35,6 +35,16 @@ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob. // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), 1 = head, 2 = window. uint32_t kv_residency_mode; + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) — + // see docs/features/rolling_kv_compute_reserve.md. INTERNAL (not user params): + // set by the llama_context two-pass measurement, NOT from llama_context_params. + // kv_compute_reserve_mib = MEASURED GPU compute-buffer size to hold back from the + // resident-window budget (0 = use the fixed OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES + // default, i.e. pass 1 / two-pass disabled). kv_window_measure_pass forces the + // window sizer to a minimal resident window (tiny device KV, boots for ANY compute + // size) so pass 1 can measure the real compute buffer without OOM. + uint32_t kv_compute_reserve_mib; + bool kv_window_measure_pass; // opencoti F5 M3 neo — asymmetric GPU/CPU pipelining of attention. // 0 = off (M2 concat path runs; binary equivalent to upstream when // headinfer_gpu_heads_frac == 1.0). diff --git a/llama.cpp/src/llama-context.cpp b/llama.cpp/src/llama-context.cpp --- a/llama.cpp/src/llama-context.cpp +++ b/llama.cpp/src/llama-context.cpp @@ -106,6 +106,12 @@ cparams.pcie_bw_gbps = params.pcie_bw_gbps; // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob — see docs/features/rolling_kv.md cparams.kv_residency_mode = params.kv_residency_mode; + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). INTERNAL: + // the memory init below runs a measurement pass, then re-creates with the measured + // reserve; these start at the "pass 1 / default reserve" values. See + // docs/features/rolling_kv_compute_reserve.md. + cparams.kv_compute_reserve_mib = 0; + cparams.kv_window_measure_pass = false; // opencoti F5 M3 neo — see docs/features/advanced_kv.md cparams.neo_pipeline_mode = params.neo_pipeline_mode; // opencoti F5 W3 #292 fused-MoE — see docs/features/advanced_kv.md @@ -373,6 +379,18 @@ } } + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342) — + // see docs/features/rolling_kv_compute_reserve.md. The resident-window sizer must + // hold back the compute buffer (allocated AFTER the KV by sched_reserve), but its + // size scales with context and is only known post-reserve. So: PASS 1 creates a + // MINIMAL resident window (tiny device KV, boots for any compute size), sched_reserve + // measures the real GPU compute buffer, then PASS 2 re-creates the KV sized against + // (free - model - measured). Scoped to the GPU flash-attn residency-managed path so + // the common resident case is unaffected (the pass-2 result is identical there). + const bool opencoti_two_pass = !hparams.vocab_only + && cparams.flash_attn && cparams.offload_kqv + && !model.devices.empty() && cparams.kv_residency_mode != 1; + // init the memory module if (!hparams.vocab_only) { llama_memory_params params_mem = { @@ -382,6 +400,8 @@ /*.ctx_type= */ cparams.ctx_type, }; + // opencoti bug-1342: pass 1 forces a minimal resident window (measure mode). + cparams.kv_window_measure_pass = opencoti_two_pass; memory.reset(model.create_memory(params_mem, cparams)); } @@ -450,6 +470,41 @@ sched_reserve(); + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve (bug-1342). Pass 1 + // (above) sized a minimal resident window so sched_reserve could not OOM; the real + // GPU compute buffer is now in backend_buf_exp_size. Sum it over the GPU backends, + // re-create the KV with that measured reserve, and re-reserve. The compute buffer is + // independent of the window/tail split, so the measurement is exact for pass 2. See + // docs/features/rolling_kv_compute_reserve.md. + if (opencoti_two_pass && !model.hparams.no_alloc) { + size_t gpu_compute = 0; + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) == GGML_BACKEND_DEVICE_TYPE_GPU) { + gpu_compute += backend_buf_exp_size[i]; + } + } + // + 256 MiB safety margin over the measured worst-case pp compute buffer. + const uint32_t measured_mib = (uint32_t) (gpu_compute / (1024 * 1024)) + 256; + LLAMA_LOG_INFO("%s: rolling-kv two-pass — measured GPU compute buffer = %u MiB; " + "re-sizing resident window against it\n", __func__, measured_mib); + cparams.kv_compute_reserve_mib = measured_mib; + cparams.kv_window_measure_pass = false; + + // free the pass-1 KV (incl. its host tail) BEFORE creating pass 2 to avoid a + // transient double host-tail allocation. + memory.reset(); + llama_memory_params params_mem2 = { + /*.type_k =*/ params.type_k, + /*.type_v =*/ params.type_v, + /*.swa_full =*/ params.swa_full, + /*.ctx_type= */ cparams.ctx_type, + }; + memory.reset(model.create_memory(params_mem2, cparams)); + + sched_need_reserve = true; + sched_reserve(); + } + if (!cparams.flash_attn) { if (ggml_is_quantized(params.type_v)) { throw std::runtime_error("quantized V cache was requested, but this requires Flash Attention"); diff --git a/llama.cpp/src/llama-kv-cache.h b/llama.cpp/src/llama-kv-cache.h --- a/llama.cpp/src/llama-kv-cache.h +++ b/llama.cpp/src/llama-kv-cache.h @@ -131,6 +131,15 @@ // 0 = auto (POSITION_WINDOW when eligible, else M2 head split), // 1 = head (force M2 split), 2 = window (force POSITION_WINDOW). uint32_t kv_residency_mode, + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve + // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md. + // compute_reserve_mib: MEASURED GPU compute-buffer size to hold + // back from the resident-window budget (0 = fixed default). + // window_measure_pass: force a MINIMAL resident window (tiny + // device KV) so the context two-pass can measure the real + // compute buffer without OOM. + uint32_t compute_reserve_mib, + bool window_measure_pass, uint32_t n_seq_max, uint32_t n_pad, uint32_t n_swa, diff --git a/llama.cpp/src/llama-kv-cache.cpp b/llama.cpp/src/llama-kv-cache.cpp --- a/llama.cpp/src/llama-kv-cache.cpp +++ b/llama.cpp/src/llama-kv-cache.cpp @@ -138,6 +138,7 @@ uint32_t kv_size, uint32_t n_stream, uint32_t vram_target_mib, + size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back const std::function & filter) { // No GPU offload → nothing is "GPU-resident" and nothing to spill; the M2 // split is only meaningful for offloaded layers. Stay at vanilla. @@ -187,9 +188,10 @@ if (vram_target_mib > 0) { budget = std::min(budget, (size_t) vram_target_mib * 1024 * 1024); } - budget = budget > OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES - ? budget - OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES - : 0; + // opencoti bug-1342: hold back the MEASURED compute buffer (two-pass) instead of the + // fixed 1536 MiB guess when the context supplied it; 0 → fall back to the default. + const size_t reserve = compute_reserve_bytes ? compute_reserve_bytes : OPENCOTI_HEADINFER_AUTO_RESERVE_BYTES; + budget = budget > reserve ? budget - reserve : 0; if (kv_demand <= budget) { LLAMA_LOG_INFO("%s: auto residency = GPU_RESIDENT (KV %.0f MiB fits budget " @@ -235,10 +237,21 @@ uint32_t n_stream, uint32_t vram_target_mib, uint32_t n_pad, + size_t compute_reserve_bytes, // opencoti bug-1342: measured (or default) compute hold-back + bool measure_pass, // opencoti bug-1342: pass 1 → minimal window (tiny device KV) const std::function & filter) { if (!offload || kv_size == 0) { return kv_size; // no position-axis spill without GPU offload } + // opencoti bug-1342 two-pass: the MEASUREMENT pass forces a MINIMAL resident window + // (one FA stride, ~0.5 MiB device KV) so pass 1 boots for ANY compute-buffer size. + // The context then measures the real compute buffer and re-sizes in pass 2. Aligned + // to lcm(n_pad, FATTN_KQ_STRIDE=256) so the tiny window is a valid FA tiling. + if (measure_pass) { + const uint32_t pad0 = n_pad ? n_pad : 32; + const uint32_t walign0 = pad0 >= 256 ? pad0 : 256; + return kv_size > walign0 ? walign0 : kv_size; + } size_t per_cell = 0; // bytes for ONE position cell across all this cache's KV layers ggml_backend_dev_t dev = nullptr; @@ -327,6 +340,8 @@ uint32_t vram_target_mib, float pcie_bw_gbps, uint32_t kv_residency_mode, + uint32_t compute_reserve_mib, + bool window_measure_pass, uint32_t n_seq_max, uint32_t n_pad, uint32_t n_swa, @@ -383,9 +398,14 @@ // A non-negative frac is the manual escape hatch / upstream default, used // verbatim. Every split decision below reads eff_gpu_heads_frac, not the raw // ctor argument, so the sentinel never leaks into the allocation math. + // opencoti bug-1342: the compute-buffer reserve to hold back from the resident budget. + // compute_reserve_mib > 0 = the MEASURED GPU compute buffer (context two-pass, pass 2); + // 0 = pass 1 / two-pass disabled → the sizers fall back to the fixed default. + const size_t opencoti_compute_reserve_bytes = (size_t) compute_reserve_mib * 1024 * 1024; const float eff_gpu_heads_frac = headinfer_gpu_heads_frac < 0.0f ? opencoti_compute_auto_gpu_heads_frac(model, hparams, type_k, type_v, - v_trans, offload, kv_size, n_stream, vram_target_mib, filter) + v_trans, offload, kv_size, n_stream, vram_target_mib, + opencoti_compute_reserve_bytes, filter) : headinfer_gpu_heads_frac; // opencoti F5 M7 Stage 3c — position-axis window sizing. Computed + logged now so the @@ -395,7 +415,7 @@ // value is intentionally unused downstream at this sub-stage. See docs/features/advanced_kv.md. const uint32_t window_cells = opencoti_compute_resident_window_cells( model, hparams, type_k, type_v, v_trans, offload, kv_size, n_stream, - vram_target_mib, n_pad, filter); + vram_target_mib, n_pad, opencoti_compute_reserve_bytes, window_measure_pass, filter); // opencoti F5 M7 Stage 4 (#338) — position-window mode is driven by the // --kv-residency-mode knob (cparams.kv_residency_mode): 0=auto, 1=head, 2=window. // The ELIGIBILITY predicate is the validated POSITION_WINDOW class: the sizer found a diff --git a/llama.cpp/src/llama-kv-cache-iswa.h b/llama.cpp/src/llama-kv-cache-iswa.h --- a/llama.cpp/src/llama-kv-cache-iswa.h +++ b/llama.cpp/src/llama-kv-cache-iswa.h @@ -41,6 +41,12 @@ // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency-tactic knob; // forwarded to both kv_base and kv_swa. 0 = auto, 1 = head, 2 = window. uint32_t kv_residency_mode, + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve + // (bug-1342); forwarded to both kv_base and kv_swa (only the + // overflowing base cache windows). See + // docs/features/rolling_kv_compute_reserve.md. + uint32_t compute_reserve_mib, + bool window_measure_pass, uint32_t n_seq_max, uint32_t n_ubatch, uint32_t n_pad, diff --git a/llama.cpp/src/llama-kv-cache-iswa.cpp b/llama.cpp/src/llama-kv-cache-iswa.cpp --- a/llama.cpp/src/llama-kv-cache-iswa.cpp +++ b/llama.cpp/src/llama-kv-cache-iswa.cpp @@ -26,6 +26,8 @@ uint32_t vram_target_mib, float pcie_bw_gbps, uint32_t kv_residency_mode, + uint32_t compute_reserve_mib, + bool window_measure_pass, uint32_t n_seq_max, uint32_t n_ubatch, uint32_t n_pad, @@ -70,6 +72,7 @@ model, type_k, type_v, v_trans, offload, unified, size_base, kv_size_initial, slot_shrink_idle_ms, headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + compute_reserve_mib, window_measure_pass, n_seq_max, n_pad, 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse, sparse_attn_block_size); LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); @@ -78,6 +81,7 @@ model, type_k, type_v, v_trans, offload, unified, size_swa, kv_size_initial, slot_shrink_idle_ms, headinfer_gpu_heads_frac, vram_target_mib, pcie_bw_gbps, kv_residency_mode, + compute_reserve_mib, window_measure_pass, n_seq_max, n_pad, hparams.n_swa, hparams.swa_type, filter_swa, reuse, sparse_attn_block_size); } diff --git a/llama.cpp/src/llama-model.cpp b/llama.cpp/src/llama-model.cpp --- a/llama.cpp/src/llama-model.cpp +++ b/llama.cpp/src/llama-model.cpp @@ -2100,6 +2100,10 @@ cparams.pcie_bw_gbps, // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob cparams.kv_residency_mode, + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve + // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md + cparams.kv_compute_reserve_mib, + cparams.kv_window_measure_pass, cparams.n_seq_max, cparams.n_ubatch, 1, @@ -2130,6 +2134,10 @@ cparams.pcie_bw_gbps, // opencoti F5 M7 Rolling KV — Stage 4 (#338) residency knob cparams.kv_residency_mode, + // opencoti F5 M7 Rolling KV — two-pass compute-buffer reserve + // (bug-1342) — see docs/features/rolling_kv_compute_reserve.md + cparams.kv_compute_reserve_mib, + cparams.kv_window_measure_pass, cparams.n_seq_max, 1, hparams.n_swa, diff --git a/llama.cpp/src/llama-memory-hybrid.cpp b/llama.cpp/src/llama-memory-hybrid.cpp --- a/llama.cpp/src/llama-memory-hybrid.cpp +++ b/llama.cpp/src/llama-memory-hybrid.cpp @@ -55,6 +55,9 @@ 0.0f, // opencoti F5 M7 Stage 4 — hybrid memory does not thread the residency-mode knob; 0 = auto. 0, + // opencoti F5 M7 bug-1342 — hybrid memory does not use the window two-pass; 0 / false. + 0, + false, n_seq_max, n_pad, n_swa, diff --git a/llama.cpp/src/llama-memory-hybrid-iswa.cpp b/llama.cpp/src/llama-memory-hybrid-iswa.cpp --- a/llama.cpp/src/llama-memory-hybrid-iswa.cpp +++ b/llama.cpp/src/llama-memory-hybrid-iswa.cpp @@ -56,6 +56,9 @@ 0.0f, // opencoti F5 M7 Stage 4 — hybrid-iswa does not thread the residency-mode knob; 0 = auto. 0, + // opencoti F5 M7 bug-1342 — hybrid-iswa does not use the window two-pass; 0 / false. + 0, + false, n_seq_max, n_ubatch, n_pad,