From: opencoti
Subject: [PATCH 0075] F5 M6 PolyKV S2 — D=512 fused FA-VEC turbo2/turbo3 (Gemma-4 GLOBAL)

#411. The Level-B fused FA-VEC turbo read (0073) covered head_dim {128,256} only;
the Gemma-4 A4B GLOBAL layers use head_dim 512 (key_length=512) and therefore fell
back to the Level-A turbo->f16 materialize — a decode tax that scales with the
global-cache n_kv. This patch adds the head_dim-512 fused FA-VEC instances for
turbo2/turbo3 so the global cache stays IN-REGISTER:

  - fattn-vec.cuh: extern DECL_FATTN_VEC_CASE(512, turbo2/3) (the kernel is already
    D-generic: nthreads/nthreads_V derive from D arithmetically, KQ smem ~4-8KB,
    VKQ ~16 regs at ncols=1 — well within budget; verified compile + run clean).
  - fattn.cu: turbo kernel-selection relax (Q->ne[0]==512 routes to VEC) + the two
    D=512 FATTN_VEC_CASE dispatch lines for turbo2/turbo3.
  - template-instances/fattn-vec-instance-turbo{2,3}_0: emit the D=512 case; the
    autogenerator (generate_cu_files.py) emits the same line so re-gen stays exact.
  - llama-graph.cpp: build_attn_mha fuse gate extended (fused_turbo_512) so D=512
    turbo2/turbo3 decode ubatches fuse; turbo4/turbo8 @ 512 keep Level-A.

Correctness is Parseval-EXACT: the WHT is block-diagonal over 128-blocks, so D=512
decomposes into 4 independent Parseval-exact 128-blocks (WHT(Q/s).K_stored ==
Q.K_true). Acceptance = LOGIT-EQUIVALENCE fused-vs-materialize (#355 harness),
real_frac=0.0 for BOTH turbo3 and turbo2 (bit-equivalent, not merely close).
Perf (the point): turbo3 @ ctx 27.7k, fused 41.0 vs materialize 29.7 tok/s = +38%,
scaling with n_kv. Gate: .opencoti/p411-d512-gate.sh (+ p411-perf.sh).

diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh
@@ -618,3 +618,8 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16)
 EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO2_0)
 EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO3_0)
 EXTERN_DECL_FATTN_VEC_TURBO(GGML_TYPE_TURBO4_0)
+
+// opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY — Gemma-4 GLOBAL
+// layers (key_length=512). Definitions in the turbo2/turbo3 instance .cu files. turbo4 stays ≤256.
+extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
+extern DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn.cu b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
--- a/llama.cpp/ggml/src/ggml-cuda/fattn.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/fattn.cu
@@ -337,6 +337,12 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t
     FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
     FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0)
 
+    // opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL
+    // layers). turbo4/turbo8 keep no D=512 instance (Level-A materialize). Matches the fattn.cu
+    // kernel-selection relax + the L3 graph gate (both turbo2/3 + D==512 only).
+    FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0)
+    FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
+
     GGML_ABORT("fatal error");
 }
 
@@ -474,7 +480,13 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     // read turbo blocks. The graph (build_attn_mha, L3) only emits turbo K/V to FLASH_ATTN_EXT when
     // it intends this path (head_dim ≤ 256, decode); turbo8 / head_dim 512 take the materialize lift.
     if (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0 || K->type == GGML_TYPE_TURBO4_0) {
-        return can_use_vector_kernel ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE;
+        // opencoti #411 (F5 M6-S2 Level-B): turbo2/3 also ship a D=512 fused VEC instance for Gemma-4
+        // GLOBAL layers (key_length=512) so the 256k global cache reads in-register instead of the
+        // Level-A turbo→f16 materialize. turbo4/turbo8 have no D=512 instance → keep the ≤256 cap
+        // (Level-A fallback at 512). The L3 graph gate only emits turbo2/3 K/V to FA at 512 to match.
+        const bool turbo512_vec = Q->ne[0] == 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 &&
+            (K->type == GGML_TYPE_TURBO2_0 || K->type == GGML_TYPE_TURBO3_0);
+        return (can_use_vector_kernel || turbo512_vec) ? BEST_FATTN_KERNEL_VEC : BEST_FATTN_KERNEL_NONE;
     }
 
     // If Turing tensor cores are available, use them:
diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
@@ -6,3 +6,4 @@
 DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
 DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
 DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
+DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512
diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
@@ -6,3 +6,4 @@
 DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
 DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
 DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
+DECL_FATTN_VEC_CASE(512, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512
diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
--- a/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
@@ -79,6 +79,10 @@ for type_k in TYPES_KV:
 for type_t in ["GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0", "GGML_TYPE_TURBO4_0"]:
     with open(f"fattn-vec-instance-{get_short_name(type_t)}-{get_short_name(type_t)}.cu", "w") as f:
         f.write(SOURCE_FATTN_VEC.format(type_k=type_t, type_v=type_t))
+        # opencoti #411 (F5 M6-S2 Level-B): head_dim 512 fused VEC for turbo2/3 ONLY (Gemma-4 GLOBAL
+        # layers, key_length=512). turbo4 has no D=512 instance (Level-A materialize fallback).
+        if type_t in ("GGML_TYPE_TURBO2_0", "GGML_TYPE_TURBO3_0"):
+            f.write(f"DECL_FATTN_VEC_CASE(512, {type_t}, {type_t}); // opencoti #411 (F5 M6-S2 Level-B): Gemma-4 GLOBAL head_dim 512\n")
 
 for ncols in [8, 16, 32, 64]:
     for ncols2 in [1, 2, 4, 8, 16, 32]:
diff --git a/llama.cpp/src/llama-graph.cpp b/llama.cpp/src/llama-graph.cpp
--- a/llama.cpp/src/llama-graph.cpp
+++ b/llama.cpp/src/llama-graph.cpp
@@ -2019,10 +2019,17 @@ ggml_tensor * llm_graph_context::build_attn_mha(
     }();
     const int64_t tbv_n_stream    = k->ne[3] > 0 ? k->ne[3] : 1;
     const int64_t tbv_nq_per_strm = q->ne[2] / tbv_n_stream;
+    // opencoti F5 M6-S2 Level-B #411: head_dim 512 (Gemma-4 GLOBAL layers, key_length=512) fuses
+    // for turbo2/turbo3 ONLY — the D=512 fused FA-VEC turbo instances added in #411 keep the 256k
+    // global cache in-register instead of the Level-A turbo→f16 materialize (a ~0.55× decode tax that
+    // scales with n_kv). turbo4/turbo8 at 512 still take the materialize fallback (no D=512 instance).
+    // The WHT is block-diagonal over 128-blocks, so 512 = 4 Parseval-exact 128-blocks → exact.
+    const bool fused_turbo_512 =
+        k->ne[0] == 512 && (k->type == GGML_TYPE_TURBO2_0 || k->type == GGML_TYPE_TURBO3_0);
     const bool fused_turbo =
         cparams.flash_attn && kq_b == nullptr &&
         is_turbo_vec(k->type) && k->type == v->type &&
-        (k->ne[0] == 128 || k->ne[0] == 256) &&
+        (k->ne[0] == 128 || k->ne[0] == 256 || fused_turbo_512) &&
         tbv_nq_per_strm <= tbv_fuse_max_nq;
 
     if (fused_turbo) {