tepirale/Ornith-Agents-A1-3.7-35B-A3B-dare_ties_v4-MTP-GGUF

Quants

Quant	Tamaño
Q4_K_M	20.5 GiB
Q5_K_M	23.9 GiB
Q6_K	27.4 GiB
Q8_0	37.8 GiB

auto detect GPU - build_llamacpp_auto.sh

#!/usr/bin/env bash
# build_llamacpp_auto.sh
# Compila llama.cpp (llama-cli + llama-quantize) detectando automáticamente
# la arquitectura CUDA de las GPUs presentes (A40=86, RTX 4090=89, A100=80,
# H100=90, RTX PRO 6000 Blackwell=120, etc.)
#
# Uso:
#   ./build_llamacpp_auto.sh                 # desde la raíz de llama.cpp
#   ./build_llamacpp_auto.sh /ruta/llama.cpp # o pasando la ruta
#   CUDA_ARCHS=86 ./build_llamacpp_auto.sh   # forzar arquitectura manualmente

set -euo pipefail

LLAMA_DIR="${1:-.}"
BUILD_DIR="build-cuda"
TARGETS="llama-cli llama-quantize llama-server"

cd "$LLAMA_DIR"

if [[ ! -f CMakeLists.txt ]]; then
  echo "ERROR: no se encontró CMakeLists.txt en $(pwd). ¿Estás en la raíz de llama.cpp?" >&2
  exit 1
fi

# ---------------------------------------------------------------
# 1. Detectar arquitecturas CUDA
# ---------------------------------------------------------------
detect_archs() {
  # Permitir override manual: CUDA_ARCHS=86 o CUDA_ARCHS="80;86"
  if [[ -n "${CUDA_ARCHS:-}" ]]; then
    echo "$CUDA_ARCHS"
    return 0
  fi

  # Método principal: nvidia-smi (drivers >= 470 soportan compute_cap)
  if command -v nvidia-smi &>/dev/null; then
    local caps
    caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null \
           | tr -d ' ' | grep -E '^[0-9]+\.[0-9]+$' || true)
    if [[ -n "$caps" ]]; then
      # "8.6" -> "86", "12.0" -> "120"; deduplicar y unir con ';'
      echo "$caps" | tr -d '.' | sort -un | paste -sd ';' -
      return 0
    fi
  fi

  # Fallback 1: CMake >= 3.24 acepta "native" (nvcc detecta la GPU local)
  local cmake_ver
  cmake_ver=$(cmake --version | head -n1 | grep -oE '[0-9]+\.[0-9]+' | head -n1)
  if [[ -n "$cmake_ver" ]] && awk -v v="$cmake_ver" 'BEGIN{split(v,a,"."); exit !(a[1]>3 || (a[1]==3 && a[2]>=24))}'; then
    echo "native"
    return 0
  fi

  return 1
}

ARCHS=$(detect_archs) || {
  echo "ERROR: no pude detectar la arquitectura de la GPU." >&2
  echo "  - Verifica que nvidia-smi funcione, o" >&2
  echo "  - Fuerza el valor: CUDA_ARCHS=86 $0" >&2
  exit 1
}

# ---------------------------------------------------------------
# 2. Info y validaciones
# ---------------------------------------------------------------
echo "=== GPUs detectadas ==="
nvidia-smi --query-gpu=name,compute_cap,memory.total --format=csv,noheader 2>/dev/null || echo "(nvidia-smi no disponible)"
echo "CMAKE_CUDA_ARCHITECTURES=$ARCHS"

# Advertencia: sm_120 (Blackwell) requiere CUDA 12.8+
if [[ "$ARCHS" == *"120"* ]] && command -v nvcc &>/dev/null; then
  CUDA_VER=$(nvcc --version | grep -oE 'release [0-9]+\.[0-9]+' | grep -oE '[0-9]+\.[0-9]+')
  if awk -v v="$CUDA_VER" 'BEGIN{split(v,a,"."); exit !(a[1]<12 || (a[1]==12 && a[2]<8))}'; then
    echo "ADVERTENCIA: sm_120 (Blackwell) requiere CUDA >= 12.8, tienes $CUDA_VER" >&2
  fi
fi

# ---------------------------------------------------------------
# 3. Compilar
# ---------------------------------------------------------------
rm -rf "$BUILD_DIR"
cmake -B "$BUILD_DIR" \
  -DGGML_CUDA=ON \
  -DCMAKE_CUDA_ARCHITECTURES="$ARCHS" \
  -DBUILD_SHARED_LIBS=OFF \
  -DLLAMA_CURL=OFF \
  > /dev/null

cmake --build "$BUILD_DIR" --config Release -j "$(nproc)" --target $TARGETS > /dev/null

echo ""
echo "✅ Compilado para arquitectura(s): $ARCHS"
echo "Binarios en: $(pwd)/$BUILD_DIR/bin/"
ls -lh "$BUILD_DIR/bin/" | grep -E 'llama-(cli|quantize)' || true

Download models

hf download tepirale/Ornith-Agents-A1-3.7-35B-A3B-dare_ties_v4-MTP-GGUF \
  --include "Ornith-Agents-A1-3.7-35B-A3B-dare_ties_v4-mtp-sidecar.gguf" \ # optional
  --include "mmproj-F32.gguf" \  # optional
  --local-dir /workspace/models/Ornith-Agents-A1-3.7-35B-A3B-dare_ties_v4-MTP-GGUF

Use (llama.cpp optional MTP and image)

./build-cuda/bin/llama-server \
  -m /content/work/gguf/Ornith-Agents-A1-3.7-35B-A3B-dare_ties_v4-Q8_0.gguf \
  --mmproj /content/work/gguf/mmproj-F32.gguf \
  --image-min-tokens 1024 \
  -ngl 999 \
  -c 51504 \
  --slots \
  # --parallel 2 \
  --slots 1
  --spec-type draft-mtp \
  --spec-draft-n-max 3 \
  --spec-draft-n-min 1 \
  --host 0.0.0.0 \
  --port 8080 \
  --flash-attn on \
  --batch-size 4096 \
  --ubatch-size 1024 \
  --api-key tu-clave-secreta \
  --dry-multiplier 0.8 \
  --spec-draft-p-min 0.7 \
  --temp 1 \
  --top-p 0.95 \
  --top-k 20 \
  --min-p 0.0 \
  --presence-penalty 0.0 \
  --repeat-penalty 1.0 \
  --parallel 1 \
  --log-verbosity 4 \
  --cache-idle-slots \
  --reasoning-preserve \
  --no-mma \
  --alias "Ornith-Agents-A1-v4" \
  --metrics

Note:

• Adding '- Don't hallucinate.' to the system file greatly improves the quality of the responses
• "--batch-size 4096" A higher value means slower decoding, but a lower value means much faster decoding. "--batch-size 1024"

Note 2:

• This model is more logical than version 1.
• Although it has MTP, it doesn't reach +250 tokens/s.
• No changes were applied to the model layers before the conversion to GGUF, only for testing purposes.
• Although this model was NOT trained, the merge took more than a day, processing approximately +500,000 graphs.

With these points in mind, I can improve the next model using dare-ties-v2, which is not in Mergekit.

Note:

• I think I've messed with Fable now, because if I ask it to do something with MergeKit to improve it, it jumps to a lower-tier model. :'(
• Fable Sometimes he tells me that the 'Qwen' models are boring and that I should work with 'gemma 4'. Just working with Qwen models or variants of the 'Fable' model can lead me down paths that are not correct.