Use CUDA llama.cpp server image
Browse files- Dockerfile +4 -4
Dockerfile
CHANGED
|
@@ -1,17 +1,17 @@
|
|
| 1 |
-
FROM ghcr.io/ggml-org/llama.cpp:server
|
| 2 |
|
| 3 |
ENV MODEL_REPO=sayshara/nemotron-3-nano-4b-ascii-art-GGUF
|
| 4 |
ENV MODEL_FILE=nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
|
| 5 |
ENV PORT=7860
|
| 6 |
ENV CTX_SIZE=2048
|
| 7 |
-
ENV N_GPU_LAYERS=
|
| 8 |
-
ENV
|
| 9 |
|
| 10 |
USER root
|
| 11 |
RUN apt-get update \
|
| 12 |
&& apt-get install -y --no-install-recommends python3 python3-pip ca-certificates \
|
| 13 |
&& rm -rf /var/lib/apt/lists/* \
|
| 14 |
-
&& python3 -m pip install --break-system-packages --no-cache-dir "huggingface_hub
|
| 15 |
|
| 16 |
COPY start_server.py /app/start_server.py
|
| 17 |
|
|
|
|
| 1 |
+
FROM ghcr.io/ggml-org/llama.cpp:server-cuda
|
| 2 |
|
| 3 |
ENV MODEL_REPO=sayshara/nemotron-3-nano-4b-ascii-art-GGUF
|
| 4 |
ENV MODEL_FILE=nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
|
| 5 |
ENV PORT=7860
|
| 6 |
ENV CTX_SIZE=2048
|
| 7 |
+
ENV N_GPU_LAYERS=99
|
| 8 |
+
ENV HF_XET_HIGH_PERFORMANCE=1
|
| 9 |
|
| 10 |
USER root
|
| 11 |
RUN apt-get update \
|
| 12 |
&& apt-get install -y --no-install-recommends python3 python3-pip ca-certificates \
|
| 13 |
&& rm -rf /var/lib/apt/lists/* \
|
| 14 |
+
&& python3 -m pip install --break-system-packages --no-cache-dir "huggingface_hub"
|
| 15 |
|
| 16 |
COPY start_server.py /app/start_server.py
|
| 17 |
|