From: opencoti Subject: [PATCH 0085] DCA quant-KV prefill — in-launch single-pass dequant (#554) Move scalar/turbo K/V dequant for the multi-chunk DCA fused flash-attn op out of the two-stage whole-cache graph lift (dca_lift_to_f16: q->f32->f16 materialized every forward, GPU-memory-bound and O(cache) — catastrophic at >=512k) and into a single-pass to_fp16_nc pool convert inside the fused launch (mirrors the non-DCA launch_fattn fast path). Byte-identical to the prior lift (both compute d*q in f32 then round to f16); restores DCA + quantized-KV as a first-class long-context path (~12x at 85k, ~30x at 512k: 46 -> 1379+ tok/s, the 1M q5_0/q4_0 path). Also moves the regime==3 fused early-out above the f16-K asserts in ggml_cuda_flash_attn_ext_lse so quant-K cells reach the fused path instead of asserting. dca_is_inkernel_liftable() gates the defer to Q8_0/Q4_0/Q4_1/Q5_0/Q5_1/Q6_0 (+ the turbo/TCQ families via the existing to_fp16_nc table); f16/bf16 keep the stock path. See .wolf/buglog.json bug-720. diff --git a/llama.cpp/src/dca.cpp b/llama.cpp/src/dca.cpp --- a/llama.cpp/src/dca.cpp +++ b/llama.cpp/src/dca.cpp @@ -204,6 +204,20 @@ return ggml_cast(ctx0, x, GGML_TYPE_F16); } +// opencoti F5 dca (DCA all-KV C-perf, bug-720): can the FUSED multi-chunk kernel dequant this cache type +// to f16 INSIDE its launch (single-pass to_fp16_nc into a transient pool buffer, mirroring launch_fattn), +// instead of dca_lift_to_f16 whole-cache materializing it to f16 in the GRAPH every forward? That graph +// lift is the prefill cliff: at 98k/q5_0 the two-stage q->f32->f16 cast ran every ubatch and dragged +// DCA-on prefill to ~138 tps vs ~1212 for the same scalar-quant cache DCA-off (which already converts +// per-op via launch_fattn). Restrict to the SCALAR quants with a to_fp16_nc converter (convert.cu:843); +// turbo/TCQ/bf16 keep dca_lift_to_f16 (already a single-pass DIRECT ->f16 cast, not the slow two-stage), +// f16 is a no-op. The in-op convert is byte-IDENTICAL to the lift (both compute d*q in f32 then round to +// f16), so this is a pure perf change, gateable by logit-equivalence. +static bool dca_is_inkernel_liftable(enum ggml_type t) { + return t == GGML_TYPE_Q8_0 || t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 || + t == GGML_TYPE_Q5_0 || t == GGML_TYPE_Q5_1 || t == GGML_TYPE_Q6_0; +} + ggml_tensor * llm_graph_context::build_attn_dca_core( const llama_kv_cache_context * mctx_cur, llm_graph_input_dca * inp_dca, @@ -226,24 +240,53 @@ // the kernel). At f16 this is a no-op (byte-identical to the prior assert path). Context-shift // stays correct independently: the modular K-shift re-ropes the STORED, still-quantized cache // via build_rope_shift's existing dequant->rope->requant quantized branch (llama-kv-cache.cpp). - ggml_tensor * k_dca = dca_lift_to_f16(ctx0, mctx_cur->get_k(ctx0, il)); + ggml_tensor * k_raw = mctx_cur->get_k(ctx0, il); + ggml_tensor * v_raw = mctx_cur->get_v(ctx0, il); + + // opencoti F5 dca (DCA all-KV C-perf, bug-720): on the analytical-band (multi-chunk) path the fused + // kernel dequants scalar-quant K/V to f16 IN ITS LAUNCH (single-pass to_fp16_nc, see + // ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case), so pass the quantized cache THROUGH instead of + // whole-cache materializing it to f16 via dca_lift_to_f16 every forward (the prefill cliff). Gate: + // - !single_chunk: only the fused op carries the in-launch convert; single_chunk uses stock + // ggml_flash_attn_ext (launch_fattn already converts scalar-quant natively) and runs only while + // n_kv <= chunk (a tiny cache) so its lift cost is negligible — keep it on the safe lift path. + // - dca_is_inkernel_liftable: scalar quants the fused launch's to_fp16_nc can read. + // - !v_trans: a transposed quant V view breaks the dim-0 (block) contiguity to_fp16_nc requires; the + // permute(0,2,1,3) below keeps dim-0 intact, but a transpose would not — fall back to the lift. + // The in-op convert is byte-identical to dca_lift_to_f16, so deferred output == lifted output. + const bool v_trans_raw = v_raw->nb[1] > v_raw->nb[2]; + const bool defer_quant_lift = + !inp_dca->single_chunk && + dca_is_inkernel_liftable(k_raw->type) && + dca_is_inkernel_liftable(v_raw->type) && + !v_trans_raw; // (2) read V once; permute K + V to the flash-attn layout [n_embd_head, seq, n_head, n_stream]. // opencoti F5 dca #444 (DCA all-KV C1): lift a block/scalar-quantized V (q8_0/turbo/...) to f16 // while it is still CONTIGUOUS — a permuted view of a block type is not cleanly cpy-able (the // block stride spans dim-0), mirroring build_attn_mha's pre-permute turbo cast. f32 V keeps the // post-permute cast below (unchanged); f16 V is untouched (byte-identical). - ggml_tensor * v = mctx_cur->get_v(ctx0, il); - if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { - v = dca_lift_to_f16(ctx0, v); - } - const bool v_trans = v->nb[1] > v->nb[2]; - ggml_tensor * vp = ggml_permute(ctx0, v, 0, 2, 1, 3); - if (v_trans) { - vp = ggml_transpose(ctx0, vp); - } - if (vp->type == GGML_TYPE_F32) { - vp = ggml_cast(ctx0, vp, GGML_TYPE_F16); + ggml_tensor * k_dca; + ggml_tensor * vp; + if (defer_quant_lift) { + // pass scalar-quant K/V straight to the fused op; permute(0,2,1,3) is a clean view (dim-0 = the + // block axis, untouched) that the launch's to_fp16_nc dequants with the permuted strides. + k_dca = k_raw; + vp = ggml_permute(ctx0, v_raw, 0, 2, 1, 3); + } else { + k_dca = dca_lift_to_f16(ctx0, k_raw); + ggml_tensor * v = v_raw; + if (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32) { + v = dca_lift_to_f16(ctx0, v); + } + const bool v_trans = v->nb[1] > v->nb[2]; + vp = ggml_permute(ctx0, v, 0, 2, 1, 3); + if (v_trans) { + vp = ggml_transpose(ctx0, vp); + } + if (vp->type == GGML_TYPE_F32) { + vp = ggml_cast(ctx0, vp, GGML_TYPE_F16); + } } ggml_tensor * kp = ggml_permute(ctx0, k_dca, 0, 2, 1, 3); diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu --- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu +++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu @@ -788,6 +788,17 @@ const int64_t n_batch = Q->ne[3]; const int64_t n_rows = n_head * n_q * n_batch; GGML_ASSERT(dst->ne[0] == DV + 1 && "flash_attn_ext_lse dst must pack [O|lse] on ne[0]"); + + // opencoti F5 dca (DCA all-KV C-perf, bug-720): the FUSED regime (op_params[4]==3) self-contains its + // attention + normalization and now dequant-on-lifts a scalar-quantized K/V INSIDE its own launch + // (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case). It never runs the streaming_lse recompute below, so + // route it out BEFORE the recompute-only f32-Q / f16-K asserts (the streaming_lse_kernel reads K as f16, + // hence those gates). Without this early-out a quantized DCA cache aborts here at line ~792. + if (ggml_get_op_params_i32(dst, 4) == 3) { + ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst); + return; + } + GGML_ASSERT(Q->type == GGML_TYPE_F32 && "flash_attn_ext_lse expects f32 Q for the lse recompute"); GGML_ASSERT(K->type == GGML_TYPE_F16 && "flash_attn_ext_lse expects f16 K for the lse recompute"); @@ -820,19 +831,11 @@ const int dca_chunk = ggml_get_op_params_i32(dst, 3); const int dca_regime = ggml_get_op_params_i32(dst, 4); - // opencoti F5 dca perf (T87.pD Branch B): FUSED single-pass DCA. The three band passes (INTER/SUCC/ - // INTRA) are folded into ONE block-per-tile kernel that streams K/V once and swaps the pre-roped Q - // variant at the 2 phase boundaries — no per-regime LSE emit, no host combine, no streaming_lse - // fallback. The fused case reads its inputs straight off dst (src[0]=Q_intra, src[1]=K, src[2]=V, - // src[3]=mask_intra, src[4]=pos_q_real, src[5]=Q_succ, src[6]=Q_inter, op_params[3]=chunk) and writes - // normalized O into the leading DV*n_rows of dst's [O|lse] buffer (the trailing lse row is left - // untouched — dca.cpp slices it off). Bands partition the causal range [0,i] ⇒ one online-softmax - // pass is bit-faithful to the 3-op + host LSE combine it replaces. - if (dca_regime == 3) { - ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch(ctx, dst); - return; - } - + // opencoti F5 dca perf (T87.pD Branch B): the FUSED single-pass DCA regime (op_params[4]==3) — three + // band passes (INTER/SUCC/INTRA) folded into ONE block-per-tile kernel, no per-regime LSE emit / host + // combine / streaming_lse fallback — is dispatched at the TOP of this function (bug-720), before the + // f32-Q / f16-K recompute asserts, so a quantized DCA cache reaches its in-launch dequant. Only the + // non-fused (regime 0/1/2) lse-recompute path continues below. float * lse_out = (float *) dst->data + DV * n_rows; opencoti_fattn_dst_lse = lse_out; opencoti_fattn_dst_lse_written = false; diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh --- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -2360,7 +2360,14 @@ const ggml_tensor * Qi = dst->src[6]; // Q_inter const ggml_tensor * Ms = dst->src[7]; // opencoti F5 dca (#617B): dense SUCC mask (NULL ⇒ fast path) const ggml_tensor * Mi = dst->src[8]; // opencoti F5 dca (#617B): dense INTER mask (NULL ⇒ fast path) - GGML_ASSERT(K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && "dca fused: f16 K/V required"); + // opencoti F5 dca (DCA all-KV C-perf, bug-720): the fused kernel reads f16 K/V, but a scalar-quantized + // cache is now dequant'd to f16 IN THIS LAUNCH below (single-pass to_fp16_nc into a transient pool + // buffer, mirroring launch_fattn) instead of being whole-cache materialized to f16 in the graph every + // forward (the prefill cliff: ~138 vs ~1212 tps at 98k/q5_0). Accept f16, or any type with a to_fp16_nc + // converter; reject anything else (the kernel would have no f16 source). + GGML_ASSERT((K->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(K->type) != nullptr) && + (V->type == GGML_TYPE_F16 || ggml_get_to_fp16_nc_cuda(V->type) != nullptr) && + "dca fused: K/V must be f16 or have a to_fp16_nc converter"); GGML_ASSERT(PQ && Qs && Qi && mask && "dca fused: missing src tensors"); // opencoti F5 dca (#617B): FRAGMENTED fallback engaged iff the host attached the dense SUCC/INTER // masks (only on a non-contiguous, post-context-shift cache). Both present or both absent. @@ -2410,6 +2417,44 @@ ggml_cuda_pool & pool = ctx.pool(); cudaStream_t main_stream = ctx.stream(); + // opencoti F5 dca (DCA all-KV C-perf, bug-720): dequant-on-lift a scalar-quantized K/V into a transient + // f16 pool buffer here — the SAME single-pass to_fp16_nc convert the stock MMA path uses (launch_fattn, + // fattn-common.cuh) — replacing the old two-stage whole-cache graph lift (dca.cpp dca_lift_to_f16, + // q->f32->f16) that ran every forward and dominated quant-KV DCA-on prefill. K/V are PERMUTED views + // (kp/vp, dim-0 = the contiguous block axis), so use the non-contiguous (strided) converter and pack to + // a contiguous f16 slot; the kernel then reads the slot with contiguous strides. f16 K/V skip the + // convert (k_data/v_data alias the tensor, strides unchanged ⇒ byte-identical to the prior f16 path). + ggml_cuda_pool_alloc K_f16(pool); + ggml_cuda_pool_alloc V_f16(pool); + const char * k_data = (const char *) K->data; + size_t nb_k1 = K->nb[1], nb_k2 = K->nb[2], nb_k3 = K->nb[3]; + const char * v_data = (const char *) V->data; + size_t nb_v1 = V->nb[1], nb_v2 = V->nb[2], nb_v3 = V->nb[3]; + if (K->type != GGML_TYPE_F16) { + const size_t ts = ggml_type_size(K->type); + GGML_ASSERT(K->nb[0] == ts && "dca fused: quant K must have contiguous dim-0 (block axis)"); + to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(K->type); + K_f16.alloc(ggml_nelements(K)); + to_fp16(k_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], + (int64_t) (nb_k1 / ts), (int64_t) (nb_k2 / ts), (int64_t) (nb_k3 / ts), main_stream); + k_data = (const char *) K_f16.ptr; + nb_k1 = K->ne[0] * sizeof(half); + nb_k2 = K->ne[1] * nb_k1; + nb_k3 = K->ne[2] * nb_k2; + } + if (V->type != GGML_TYPE_F16) { + const size_t ts = ggml_type_size(V->type); + GGML_ASSERT(V->nb[0] == ts && "dca fused: quant V must have contiguous dim-0 (block axis)"); + to_fp16_nc_cuda_t to_fp16 = ggml_get_to_fp16_nc_cuda(V->type); + V_f16.alloc(ggml_nelements(V)); + to_fp16(v_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], + (int64_t) (nb_v1 / ts), (int64_t) (nb_v2 / ts), (int64_t) (nb_v3 / ts), main_stream); + v_data = (const char *) V_f16.ptr; + nb_v1 = V->ne[0] * sizeof(half); + nb_v2 = V->ne[1] * nb_v1; + nb_v3 = V->ne[2] * nb_v2; + } + const int ntiles_x = (Q->ne[1] + ncols1 - 1) / ncols1; const int gqa_ratio = Q->ne[2] / K->ne[2]; const int ntiles_z_gqa = (gqa_ratio + ncols2 - 1) / ncols2; @@ -2490,7 +2535,7 @@ const dim3 grid(nblocks, 1, 1); kernel<<>>( - (const char *) Q->data, (const char *) K->data, (const char *) V->data, + (const char *) Q->data, k_data, v_data, // opencoti dca bug-720: f16 slot (or aliased f16 tensor) (const char *) mask->data, Ms ? (const char *) Ms->data : nullptr, // opencoti F5 dca (#617B): dense SUCC mask (null ⇒ fast path) Mi ? (const char *) Mi->data : nullptr, // opencoti F5 dca (#617B): dense INTER mask (null ⇒ fast path) @@ -2499,8 +2544,8 @@ (float *) KQV->data, dst_meta_ptr, scale, 0.0f /*max_bias*/, 1.0f /*m0*/, 1.0f /*m1*/, n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], K->nb[1], K->nb[2], K->nb[3], - V->nb[1], V->nb[2], V->nb[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb_k1, nb_k2, nb_k3, // opencoti dca bug-720: slot strides + nb_v1, nb_v2, nb_v3, mask->ne[1], mask->ne[2], mask->ne[3], mask->nb[1], mask->nb[2], mask->nb[3]); CUDA_CHECK(cudaGetLastError());