Download GGUF at runtime

Dockerfile

@@ -2,23 +2,19 @@ FROM ghcr.io/ggml-org/llama.cpp:server
 
 ENV MODEL_REPO=sayshara/nemotron-3-nano-4b-ascii-art-GGUF
 ENV MODEL_FILE=nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
-ENV MODEL_PATH=/models/nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
 ENV PORT=7860
 ENV CTX_SIZE=2048
 ENV N_GPU_LAYERS=0
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
 
 USER root
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl ca-certificates \
+    && apt-get install -y --no-install-recommends python3 python3-pip ca-certificates \
     && rm -rf /var/lib/apt/lists/* \
-    && mkdir -p /models \
-    && curl -L --fail --retry 5 --retry-delay 5 \
-       "https://huggingface.co/${MODEL_REPO}/resolve/main/${MODEL_FILE}" \
-       -o "${MODEL_PATH}"
+    && python3 -m pip install --break-system-packages --no-cache-dir "huggingface_hub[hf_transfer]"
 
-COPY start_server.sh /app/start_server.sh
-RUN chmod +x /app/start_server.sh
+COPY start_server.py /app/start_server.py
 
 EXPOSE 7860
 ENTRYPOINT []
-CMD ["/app/start_server.sh"]
+CMD ["python3", "/app/start_server.py"]

start_server.py

@@ -0,0 +1,44 @@
+from huggingface_hub import hf_hub_download
+import os
+import shutil
+import subprocess
+import sys
+
+repo = os.environ.get("MODEL_REPO", "sayshara/nemotron-3-nano-4b-ascii-art-GGUF")
+filename = os.environ.get("MODEL_FILE", "nemotron-3-nano-4b-ascii-art-q4_k_m.gguf")
+token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
+cache_dir = os.environ.get("HF_HOME", "/data/.cache/huggingface")
+local_dir = os.environ.get("MODEL_DIR", "/data/models")
+os.makedirs(local_dir, exist_ok=True)
+
+print(f"Downloading {repo}/{filename}...", flush=True)
+model_path = hf_hub_download(
+    repo_id=repo,
+    filename=filename,
+    token=token,
+    cache_dir=cache_dir,
+    local_dir=local_dir,
+)
+
+server_bin = shutil.which("llama-server")
+if server_bin is None:
+    for candidate in ("/app/llama-server", "/usr/local/bin/llama-server", "/usr/bin/llama-server", "/llama-server"):
+        if os.path.exists(candidate):
+            server_bin = candidate
+            break
+if server_bin is None:
+    print("Could not find llama-server", flush=True)
+    raise FileNotFoundError("llama-server")
+
+cmd = [
+    server_bin,
+    "-m", model_path,
+    "--host", "0.0.0.0",
+    "--port", os.environ.get("PORT", "7860"),
+    "-c", os.environ.get("CTX_SIZE", "2048"),
+    "-ngl", os.environ.get("N_GPU_LAYERS", "0"),
+]
+
+print("Starting:", " ".join(cmd), flush=True)
+sys.stdout.flush()
+subprocess.run(cmd, check=True)