RUN_RTX3090_FROM_REPO.sh

#!/usr/bin/env bash
set -euo pipefail

# Reproduce the RTX 3090 path validated for Qwen3.8-27B-IQ1_M.
# Run this from the root of the Hugging Face repository after downloading it.

MODEL="${MODEL:-$PWD/Qwen3.8-27B-IQ1_M.gguf}"
OLLAMA_MODEL="${OLLAMA_MODEL:-qwen38-iq1m}"
OLLAMA_HOST="${OLLAMA_HOST:-127.0.0.1:11434}"
PROMPT="${*:-Write one grammatical sentence of at least eight words explaining why the sky looks blue.}"

if [ ! -f "$MODEL" ]; then
  echo "Missing model: $MODEL" >&2
  exit 2
fi

EXPECTED_MODEL_SHA256=131cdf5c1c4b547081543382b00434e9ebf3f8eb369ef3714550086074f80bdf
ACTUAL_MODEL_SHA256="$(sha256sum "$MODEL" | awk '{print $1}')"
[ "$ACTUAL_MODEL_SHA256" = "$EXPECTED_MODEL_SHA256" ] || {
  echo "Model SHA256 mismatch: $ACTUAL_MODEL_SHA256" >&2
  exit 3
}

command -v nvidia-smi >/dev/null || {
  echo "nvidia-smi is required for the validated NVIDIA GPU path." >&2
  exit 4
}

if ! command -v ollama >/dev/null; then
  echo "Installing the validated Ollama version 0.32.14..." >&2
  curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.32.14 sh
fi

export OLLAMA_HOST
export OLLAMA_LLM_LIBRARY=cuda_v12
export OLLAMA_MAX_LOADED_MODELS=1
export OLLAMA_NUM_PARALLEL=1
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"

OLLAMA_VERSION_TEXT="$(ollama --version 2>&1 || true)"
echo "Using $OLLAMA_VERSION_TEXT" >&2

ollama serve >ollama-qwen38-iq1m.log 2>&1 &
SERVER_PID=$!
cleanup() { kill "$SERVER_PID" 2>/dev/null || true; }
trap cleanup EXIT

for _ in $(seq 1 150); do
  curl -fsS "http://$OLLAMA_HOST/api/version" >/dev/null 2>&1 && break
  sleep 0.2
done

MODELFILE="$(mktemp)"
trap 'rm -f "$MODELFILE"; cleanup' EXIT
cat >"$MODELFILE" <<EOF
FROM $MODEL
PARAMETER num_ctx 2048
PARAMETER temperature 0
EOF

ollama create "$OLLAMA_MODEL" -f "$MODELFILE"

python3 - "$OLLAMA_HOST" "$OLLAMA_MODEL" "$PROMPT" <<'PY'
import json, sys, urllib.request
host, model, prompt = sys.argv[1:]
payload = json.dumps({
    "model": model,
    "prompt": prompt,
    "stream": False,
    "options": {"temperature": 0, "num_ctx": 2048, "num_predict": 128},
}).encode()
req = urllib.request.Request(
    "http://" + host + "/api/generate",
    data=payload,
    headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=900) as response:
    data = json.load(response)
print(data.get("response", ""))
PY

echo >&2
echo "GPU state:" >&2
nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,power.draw --format=csv,noheader >&2