# yaml-language-server: $schema=https://raw.githubusercontent.com/basetenlabs/truss/main/truss/config.schema.json

model_name: glm-5.2-vision-nvfp4-4gpu
description: GLM-5.2-Vision NVFP4 with 256k context on 4 B200 GPUs.
model_metadata:
  example_model_input:
    model: glm-5.2-vision
    messages:
      - role: user
        content:
          - type: image_url
            image_url:
              url: "https://ultralytics.com/images/bus.jpg"
          - type: text
            text: "Describe this image in detail."
    max_tokens: 512
    temperature: 1.0
    top_p: 0.95
  tags:
    - openai-compatible

# Deploy GLM-5.2-Vision (NVFP4 build) on four B200s instead of eight.
#   uvx truss push --config config_nvfp4_4gpu.yaml
#
# 466GB of NVFP4 weights across 4x183GB leaves ~11GB/GPU of headroom, so this runs
# at mem-fraction 0.90 and a 256k context rather than the full 1M. Measured on
# 4xB200: ~172GB/GPU resident, KV budget ~973k tokens — comfortably above 256k.
#
# There is no checkpoint assembly here: the public Hugging Face repo below is
# already the fully assembled checkpoint (GLM-5.2 text + MoonViT vision tower +
# trained projector + processor remote code), so start_server.sh only installs
# the plugin and launches SGLang.
base_image:
  # SGLang 0.5.13, CUDA 13, Blackwell (sm_100). Ships the DSA sparse-attention and
  # trtllm-gen fp4-MoE kernels this checkpoint needs.
  image: lmsysorg/sglang:v0.5.13-cu130

docker_server:
  # data/ -> /app/data ; packages/ -> /packages
  start_command: bash /app/data/start_server.sh
  readiness_endpoint: /health
  liveness_endpoint: /health
  predict_endpoint: /v1/chat/completions
  server_port: 8000

model_cache:
  # Public Hugging Face repo: no Hugging Face token or Baseten secret is needed.
  - repo_id: baseten/GLM-5.2-Vision-NVFP4
    revision: main
    volume_folder: glm5v_nvfp4
    use_volume: true

environment_variables:
  # NVFP4 is Blackwell-only and needs the modelopt_fp4 path plus two disable flags
  # (see start_server.sh). GLM5V_CKPT must match volume_folder above.
  GLM5V_CKPT: /app/model_cache/glm5v_nvfp4
  GLM5V_QUANTIZATION: modelopt_fp4
  GLM5V_TP_SIZE: "4"
  GLM5V_MAX_MODEL_LEN: "262144"
  GLM5V_MEM_FRACTION: "0.90"

resources:
  accelerator: B200:4
  use_gpu: true

runtime:
  predict_concurrency: 32
  health_checks:
    # First boot pays the weight transfer + SGLang load + cuda-graph capture.
    restart_threshold_seconds: 3000
    stop_traffic_threshold_seconds: 120