#!/usr/bin/env bash
# Optimized Kimi-K3 IQ1_S REAP568 profile: 64K context, uBatch 1024.
# Uses the measured FP16-K/FP16-V cache path (+3.4–4.4% prefill).
# Set BIN_DIR, MODEL, and RPC_WORKERS for your own four-node environment.
set -euo pipefail

BIN_DIR=${BIN_DIR:-/opt/llama.cpp/build/bin}
MODEL=${MODEL:-$PWD/Kimi-K3-UD-IQ1_S-00001-of-00014.gguf}
RPC_WORKERS=${RPC_WORKERS:-10.10.10.1:50052,10.10.10.2:50052,10.10.10.4:50052}
PORT=${PORT:-8210}

exec env \
  GGML_RPC_SKIP_HASH=1 \
  CUDA_SCALE_LAUNCH_QUEUES=4x \
  LLAMA_MMAP_PREFETCH=0 \
  LLAMA_PARALLEL_DEVICE_LOAD=1 \
  "$BIN_DIR/llama-server" \
  --model "$MODEL" \
  --alias kimi-k3-iq1s-reap568-64k-ubatch1024 \
  --host 0.0.0.0 --port "$PORT" \
  --ctx-size 65536 --parallel 1 \
  --n-gpu-layers 999 \
  --rpc "$RPC_WORKERS" \
  --split-mode layer --tensor-split 1,1,1,1 \
  --flash-attn on --fit off \
  --cache-type-k f16 --cache-type-v f16 \
  --batch-size 2048 --ubatch-size 1024 \
  --threads 16 --threads-batch 20 \
  --reasoning auto --metrics