diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh --- a/llamafile/cuda.sh +++ b/llamafile/cuda.sh @@ -259,6 +259,22 @@ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS" fi +# opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles +# the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left +# alone at the build tail uses one core while the rest idle. --threads compiles the arches +# concurrently. To avoid oversubscribing the bulk (JOBS nvcc already saturate the cores), we divide +# the outer job-slot count by NVCC_THREADS so PEAK concurrency stays ≈ JOBS (slots × threads). The +# Windows cuda.bat already passes --threads 5; the Linux path lacked it. Tunable: NVCC_THREADS=1 +# disables (restores pre-#531 behavior). See docs/protocols/LLAMAFILE_UPGRADE.md. +NVCC_THREADS="${NVCC_THREADS:-2}" +JOBS_EFF="$JOBS" +if [ "${NVCC_THREADS}" -gt 1 ] 2>/dev/null; then + COMMON_FLAGS="$COMMON_FLAGS --threads ${NVCC_THREADS}" + JOBS_EFF=$(( JOBS / NVCC_THREADS )) + [ "$JOBS_EFF" -lt 1 ] && JOBS_EFF=1 +fi +echo " nvcc --threads ${NVCC_THREADS} → ${JOBS_EFF} job-slots × ${NVCC_THREADS} arch-threads (#531)" + # Collect sources collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_QUANTS" echo " Sources: $NUM_SOURCES .cu files" @@ -267,7 +283,7 @@ START_TIME=$(date +%s) # Compile GPU sources -compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS" +compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" COMPILE_TIME=$(date +%s) echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds" diff --git a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh --- a/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -2600,3 +2600,41 @@ extern DECL_FATTN_MMA_F16_CASE(576, 512, 16, 4); extern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 32); extern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32); + +// opencoti-hook: cuda-build-parallelism (#531) — shard the DCA-fused MMA case into parallel +// instance TUs (template-instances/fattn-mma-f16-dca-fused-instance-*.cu), mirroring the stock +// DECL_FATTN_MMA_F16_CASE sharding above. Before this, fattn.cu's dca-fused dispatch +// (ggml_cuda_flash_attn_ext_mma_f16_dca_fused_switch) instantiated all 48 +// ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case INLINE — DKQ==DV ∈ {128,256,512} +// × ncols2 ∈ {1,2,4,8} × ncols1 ∈ {8,16,32,64}/ncols2 → 576 device kernels in ONE TU = a ~52-min +// SOLO compile tail (fattn.o was 29 MB, the largest object, finishing ~52 min after every other TU). +// These extern decls move the instantiation to the parallel bulk; fattn.cu now just references the +// symbols (the regular MMA/TILE/VEC cases were already sharded this way; only our DCA kernel wasn't). +// The instance set matches the dispatch 1:1 — every reachable _case has a +// definition, none extra — so the linked DSO is byte-for-byte equivalent (same 576 kernels, relocated). +#define DECL_FATTN_MMA_F16_DCA_FUSED_CASE(DKQ, DV, ncols1, ncols2) \ + template void ggml_cuda_flash_attn_ext_mma_f16_dca_fused_case \ + (ggml_backend_cuda_context & ctx, ggml_tensor * dst) \ + +// DCA always uses DKQ==DV ∈ {128,256,512}; one extern per (ncols1,ncols2) across all three head dims. +#define EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(ncols1, ncols2) \ + extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, ncols1, ncols2); \ + extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, ncols1, ncols2); \ + extern DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, ncols1, ncols2); \ + +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 1) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 1) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 1) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(64, 1) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 2) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 2) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 2) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(32, 2) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 4) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 4) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 4) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ(16, 4) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 1, 8) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 2, 8) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 4, 8) +EXTERN_DECL_FATTN_MMA_F16_DCA_FUSED_ALL_DKQ( 8, 8) diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_1.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 1); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_2.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 2); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_16-ncols2_4.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 16, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 16, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 16, 4); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_1-ncols2_8.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 1, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 1, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 1, 8); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_4.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 4); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_2-ncols2_8.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 2, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 2, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 2, 8); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_1.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 1); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_32-ncols2_2.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 32, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 32, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 32, 2); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_2.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 2); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_4.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 4); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_4-ncols2_8.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 4, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 4, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 4, 8); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_64-ncols2_1.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 64, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 64, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 64, 1); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_1.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 1); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 1); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_2.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 2); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 2); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_4.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 4); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 4); diff --git a/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu new file mode 100644 --- /dev/null +++ b/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-dca-fused-instance-ncols1_8-ncols2_8.cu @@ -0,0 +1,10 @@ +// opencoti-hook: cuda-build-parallelism (#531) — DCA-fused MMA case instance, relocated out of +// fattn.cu so it compiles in the parallel bulk instead of inflating fattn.cu's solo tail. +// Mirrors template-instances/fattn-mma-f16-instance-*.cu. extern decls live in ../fattn-mma-f16.cuh. +// DKQ==DV ∈ {128,256,512} (the only head dims DCA dispatches: Gemma-4 512, qwen35moe 256, qwen* 128). + +#include "../fattn-mma-f16.cuh" + +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(128, 128, 8, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(256, 256, 8, 8); +DECL_FATTN_MMA_F16_DCA_FUSED_CASE(512, 512, 8, 8);