model_name: glm-5.2-vision-nvfp4-4gpu
model_metadata:
  example_model_input:
    model: glm-5.2-vision-nvfp4-4gpu
    messages:
      - role: user
        content:
          - type: image_url
            image_url:
              url: "https://ultralytics.com/images/bus.jpg"
          - type: text
            text: "Describe this image in detail."
    max_tokens: 512
    temperature: 1.0
    top_p: 0.95
  tags:
    - openai-compatible

# Deploy GLM-5.2-Vision (NVFP4 build) on FOUR B200s instead of eight.
#   truss push --config config_nvfp4_4gpu.yaml --remote <your-remote>
#
# 465GB of NVFP4 weights across 4x183GB leaves ~11GB/GPU of headroom, so this runs
# at mem-fraction 0.90 and a 256k context rather than the full 1M. Measured on
# 4xB200: ~172GB/GPU resident, KV budget ~973k tokens — comfortably above 256k.
#
# Unlike the internal deployments this was derived from, there is NO checkpoint
# assembly here: the model_cache repo below is already the fully assembled
# checkpoint (GLM-5.2 text + MoonViT vision tower + trained projector + processor
# remote code), so start_server.sh only installs the plugin and launches SGLang.
#
# To deploy:  truss push --remote <your-remote>
base_image:
  # SGLang 0.5.13, CUDA 13, Blackwell (sm_100). Ships the DSA sparse-attention and
  # trtllm-gen fp4-MoE kernels this checkpoint needs.
  image: lmsysorg/sglang:v0.5.13-cu130

docker_server:
  # data/ -> /app/data ; packages/ -> /packages
  start_command: bash /app/data/start_server.sh
  readiness_endpoint: /health
  liveness_endpoint: /health
  predict_endpoint: /v1/chat/completions
  server_port: 8000

model_cache:
  # The whole model, one repo — no assembly at serve time.
  - repo_id: baseten/GLM-5.2-Vision-NVFP4
    revision: main
    volume_folder: glm5v_nvfp4
    use_volume: true

environment_variables:
  # NVFP4 is Blackwell-only and needs the modelopt_fp4 path plus two disable flags
  # (see start_server.sh). GLM5V_CKPT must match volume_folder above.
  GLM5V_CKPT: /app/model_cache/glm5v_nvfp4
  GLM5V_QUANTIZATION: modelopt_fp4
  GLM5V_TP_SIZE: "4"
  GLM5V_MAX_MODEL_LEN: "262144"
  GLM5V_MEM_FRACTION: "0.90"

secrets:
  hf_access_token: null

resources:
  accelerator: B200:4
  use_gpu: true

runtime:
  predict_concurrency: 32
  health_checks:
    # First boot pays the weight transfer + SGLang load + cuda-graph capture.
    restart_threshold_seconds: 3000
    stop_traffic_threshold_seconds: 120