diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh index 4781516..5b1c26c 100644 --- a/llamafile/build-functions.sh +++ b/llamafile/build-functions.sh @@ -238,8 +238,15 @@ compile_gpu_sources_parallel() { obj="$build_dir/${base}.o" fi - # Skip if object file is newer than source - if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then + # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness. The CUDA kernels + # live in .cuh HEADERS; template-instance .cu stubs never change, so comparing the object + # only to its .cu left STALE objects on header edits → stale DSO. Also require the object + # to be newer than the newest ggml-tree header ($OPENCOTI_CUDA_DEPS_NEWEST, exported by + # cuda.sh). Compilation runs through ccache (see cuda.sh), so a header edit then recompiles + # only the TUs whose expanded includes actually changed. The `-z` guard preserves the old + # .cu-only behaviour for any caller that doesn't export the var. + if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \ + && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then echo "[$count/$total] Skipping: $base.cu (up to date)" continue fi @@ -298,8 +305,11 @@ compile_ggml_core() { local name="${base%.*}" local obj="$build_dir/ggml-core-${name}.o" - # Skip if object file is newer than source - if [ -f "$obj" ] && [ "$obj" -nt "$src" ]; then + # opencoti-hook: cuda-build-ccache (bug-858) — HEADER-AWARE freshness (see cuda.sh). Skip + # only when the object is newer than the source AND every ggml-tree header, so a header + # edit correctly forces a recompile. `-z` guard keeps the old behaviour if the var is unset. + if [ -f "$obj" ] && [ "$obj" -nt "$src" ] \ + && { [ -z "$OPENCOTI_CUDA_DEPS_NEWEST" ] || [ "$obj" -nt "$OPENCOTI_CUDA_DEPS_NEWEST" ]; }; then echo " Skipping: $base (up to date)" continue fi diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh index 18d4357..8e38209 100755 --- a/llamafile/cuda.sh +++ b/llamafile/cuda.sh @@ -293,10 +293,29 @@ collect_gpu_sources "$GGML_CUDA_DIR" "$EXTRA_SOURCES" "$NO_IQ_QUANTS" "$FA_ALL_Q echo " Sources: $NUM_SOURCES .cu files" echo "" +# opencoti-hook: cuda-build-ccache (bug-858) — see docs/protocols/LLAMAFILE_UPGRADE.md. +# The CUDA kernels live in .cuh HEADERS (fattn-vec.cuh, fattn-mma-f16.cuh, ...); the +# template-instance .cu stubs that #include them essentially never change. The per-TU +# incremental skip in build-functions.sh compared each object ONLY to its .cu file, so editing a +# .cuh recompiled NOTHING → a silently STALE DSO (bug-858; same class as the host bug-167 -MMD +# fix, which was never applied to this hand-rolled CUDA loop). Fix: (1) route every TU compile +# through ccache, whose include-manifest invalidates the cached object when the .cu OR any +# included header changes; (2) export the newest header mtime in the ggml tree so the no-ccache +# fallback skip is header-aware. Linking (link_shared_library) stays on raw $NVCC — ccache caches +# compiles only. +CUDA_CC="$NVCC" +if command -v ccache >/dev/null 2>&1; then + CUDA_CC="ccache $NVCC" + echo " ccache: ON for CUDA compilation (header-aware via include manifest)" +else + echo " ccache: NOT FOUND — install ccache for fast + correct incremental CUDA rebuilds" +fi +export OPENCOTI_CUDA_DEPS_NEWEST=$(find "$GGML_CUDA_DIR/../.." \( -name '*.cuh' -o -name '*.h' -o -name '*.hpp' \) -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + START_TIME=$(date +%s) # Compile GPU sources -compile_gpu_sources_parallel "$NVCC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" +compile_gpu_sources_parallel "$CUDA_CC" "$ARCH_FLAGS" "$COMMON_FLAGS" "$BUILD_DIR" "$JOBS_EFF" COMPILE_TIME=$(date +%s) echo "Compilation took $((COMPILE_TIME - START_TIME)) seconds"