diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh --- a/llamafile/cuda.sh +++ b/llamafile/cuda.sh @@ -259,6 +259,19 @@ COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_FA_ALL_QUANTS" fi +# opencoti-hook: cuda-graphs (#533 decode-tps lever L6) — enable CUDA graph capture+replay. +# Upstream llama.cpp's CMake defines GGML_CUDA_USE_GRAPHS by default; the llamafile cuda.sh build +# omitted it, so the entire `#ifdef USE_CUDA_GRAPH` capture/replay path (ggml-cuda.cu:3318/4478/ +# 4515/4593/4698; common.cuh:1218) was compiled OUT — every decode step launched hundreds of +# kernels individually with full per-launch CPU overhead on a latency-bound (~4.5% SM) decode. +# This flag turns it on; the runtime guards (split buffers, unsupported node types, the opencoti +# NEO head-split force-disable) fall back to eager capture-by-capture when a graph isn't viable, so +# it composes with the opencoti stack. Tunable: OPENCOTI_NO_CUDA_GRAPHS=1 omits it (pre-#533). +# See docs/evaluations/decode-levers.md (L6). +if [ "${OPENCOTI_NO_CUDA_GRAPHS:-0}" != "1" ]; then + COMMON_FLAGS="$COMMON_FLAGS -DGGML_CUDA_USE_GRAPHS" +fi + # opencoti-hook: cuda-build-parallelism (#531) — nvcc intra-TU arch threading. Each nvcc TU compiles # the gencode arches (sm_75/80/86/89/90/120f); without --threads they run serially, so any TU left # alone at the build tail uses one core while the rest idle. --threads compiles the arches