sayshara commited on
Commit
d22f307
·
verified ·
1 Parent(s): 1bd77b1

Use CUDA llama.cpp server image

Browse files
Files changed (1) hide show
  1. Dockerfile +4 -4
Dockerfile CHANGED
@@ -1,17 +1,17 @@
1
- FROM ghcr.io/ggml-org/llama.cpp:server
2
 
3
  ENV MODEL_REPO=sayshara/nemotron-3-nano-4b-ascii-art-GGUF
4
  ENV MODEL_FILE=nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
5
  ENV PORT=7860
6
  ENV CTX_SIZE=2048
7
- ENV N_GPU_LAYERS=0
8
- ENV HF_HUB_ENABLE_HF_TRANSFER=1
9
 
10
  USER root
11
  RUN apt-get update \
12
  && apt-get install -y --no-install-recommends python3 python3-pip ca-certificates \
13
  && rm -rf /var/lib/apt/lists/* \
14
- && python3 -m pip install --break-system-packages --no-cache-dir "huggingface_hub[hf_transfer]"
15
 
16
  COPY start_server.py /app/start_server.py
17
 
 
1
+ FROM ghcr.io/ggml-org/llama.cpp:server-cuda
2
 
3
  ENV MODEL_REPO=sayshara/nemotron-3-nano-4b-ascii-art-GGUF
4
  ENV MODEL_FILE=nemotron-3-nano-4b-ascii-art-q4_k_m.gguf
5
  ENV PORT=7860
6
  ENV CTX_SIZE=2048
7
+ ENV N_GPU_LAYERS=99
8
+ ENV HF_XET_HIGH_PERFORMANCE=1
9
 
10
  USER root
11
  RUN apt-get update \
12
  && apt-get install -y --no-install-recommends python3 python3-pip ca-certificates \
13
  && rm -rf /var/lib/apt/lists/* \
14
+ && python3 -m pip install --break-system-packages --no-cache-dir "huggingface_hub"
15
 
16
  COPY start_server.py /app/start_server.py
17