File size: 2,487 Bytes
0014a61 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | #!/usr/bin/env bash
set -euo pipefail
# Reproduce the RTX 3090 path validated for Qwen3.8-27B-IQ1_M.
# Run this from the root of the Hugging Face repository after downloading it.
MODEL="${MODEL:-$PWD/Qwen3.8-27B-IQ1_M.gguf}"
OLLAMA_MODEL="${OLLAMA_MODEL:-qwen38-iq1m}"
OLLAMA_HOST="${OLLAMA_HOST:-127.0.0.1:11434}"
PROMPT="${*:-Write one grammatical sentence of at least eight words explaining why the sky looks blue.}"
if [ ! -f "$MODEL" ]; then
echo "Missing model: $MODEL" >&2
exit 2
fi
EXPECTED_MODEL_SHA256=131cdf5c1c4b547081543382b00434e9ebf3f8eb369ef3714550086074f80bdf
ACTUAL_MODEL_SHA256="$(sha256sum "$MODEL" | awk '{print $1}')"
[ "$ACTUAL_MODEL_SHA256" = "$EXPECTED_MODEL_SHA256" ] || {
echo "Model SHA256 mismatch: $ACTUAL_MODEL_SHA256" >&2
exit 3
}
command -v nvidia-smi >/dev/null || {
echo "nvidia-smi is required for the validated NVIDIA GPU path." >&2
exit 4
}
if ! command -v ollama >/dev/null; then
echo "Installing the validated Ollama version 0.32.14..." >&2
curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.32.14 sh
fi
export OLLAMA_HOST
export OLLAMA_LLM_LIBRARY=cuda_v12
export OLLAMA_MAX_LOADED_MODELS=1
export OLLAMA_NUM_PARALLEL=1
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
OLLAMA_VERSION_TEXT="$(ollama --version 2>&1 || true)"
echo "Using $OLLAMA_VERSION_TEXT" >&2
ollama serve >ollama-qwen38-iq1m.log 2>&1 &
SERVER_PID=$!
cleanup() { kill "$SERVER_PID" 2>/dev/null || true; }
trap cleanup EXIT
for _ in $(seq 1 150); do
curl -fsS "http://$OLLAMA_HOST/api/version" >/dev/null 2>&1 && break
sleep 0.2
done
MODELFILE="$(mktemp)"
trap 'rm -f "$MODELFILE"; cleanup' EXIT
cat >"$MODELFILE" <<EOF
FROM $MODEL
PARAMETER num_ctx 2048
PARAMETER temperature 0
EOF
ollama create "$OLLAMA_MODEL" -f "$MODELFILE"
python3 - "$OLLAMA_HOST" "$OLLAMA_MODEL" "$PROMPT" <<'PY'
import json, sys, urllib.request
host, model, prompt = sys.argv[1:]
payload = json.dumps({
"model": model,
"prompt": prompt,
"stream": False,
"options": {"temperature": 0, "num_ctx": 2048, "num_predict": 128},
}).encode()
req = urllib.request.Request(
"http://" + host + "/api/generate",
data=payload,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=900) as response:
data = json.load(response)
print(data.get("response", ""))
PY
echo >&2
echo "GPU state:" >&2
nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,power.draw --format=csv,noheader >&2
|