truss deployment example

truss/config_nvfp4_4gpu.yaml

@@ -0,0 +1,73 @@
+model_name: glm-5.2-vision-nvfp4-4gpu
+model_metadata:
+  example_model_input:
+    model: glm-5.2-vision-nvfp4-4gpu
+    messages:
+      - role: user
+        content:
+          - type: image_url
+            image_url:
+              url: "https://ultralytics.com/images/bus.jpg"
+          - type: text
+            text: "Describe this image in detail."
+    max_tokens: 512
+    temperature: 1.0
+    top_p: 0.95
+  tags:
+    - openai-compatible
+
+# Deploy GLM-5.2-Vision (NVFP4 build) on FOUR B200s instead of eight.
+#   truss push --config config_nvfp4_4gpu.yaml --remote <your-remote>
+#
+# 465GB of NVFP4 weights across 4x183GB leaves ~11GB/GPU of headroom, so this runs
+# at mem-fraction 0.90 and a 256k context rather than the full 1M. Measured on
+# 4xB200: ~172GB/GPU resident, KV budget ~973k tokens — comfortably above 256k.
+#
+# Unlike the internal deployments this was derived from, there is NO checkpoint
+# assembly here: the model_cache repo below is already the fully assembled
+# checkpoint (GLM-5.2 text + MoonViT vision tower + trained projector + processor
+# remote code), so start_server.sh only installs the plugin and launches SGLang.
+#
+# To deploy:  truss push --remote <your-remote>
+base_image:
+  # SGLang 0.5.13, CUDA 13, Blackwell (sm_100). Ships the DSA sparse-attention and
+  # trtllm-gen fp4-MoE kernels this checkpoint needs.
+  image: lmsysorg/sglang:v0.5.13-cu130
+
+docker_server:
+  # data/ -> /app/data ; packages/ -> /packages
+  start_command: bash /app/data/start_server.sh
+  readiness_endpoint: /health
+  liveness_endpoint: /health
+  predict_endpoint: /v1/chat/completions
+  server_port: 8000
+
+model_cache:
+  # The whole model, one repo — no assembly at serve time.
+  - repo_id: baseten/GLM-5.2-Vision-NVFP4
+    revision: main
+    volume_folder: glm5v_nvfp4
+    use_volume: true
+
+environment_variables:
+  # NVFP4 is Blackwell-only and needs the modelopt_fp4 path plus two disable flags
+  # (see start_server.sh). GLM5V_CKPT must match volume_folder above.
+  GLM5V_CKPT: /app/model_cache/glm5v_nvfp4
+  GLM5V_QUANTIZATION: modelopt_fp4
+  GLM5V_TP_SIZE: "4"
+  GLM5V_MAX_MODEL_LEN: "262144"
+  GLM5V_MEM_FRACTION: "0.90"
+
+secrets:
+  hf_access_token: null
+
+resources:
+  accelerator: B200:4
+  use_gpu: true
+
+runtime:
+  predict_concurrency: 32
+  health_checks:
+    # First boot pays the weight transfer + SGLang load + cuda-graph capture.
+    restart_threshold_seconds: 3000
+    stop_traffic_threshold_seconds: 120

truss/data/start_server.sh

@@ -68,7 +68,7 @@ fi
 exec python3 -m sglang.launch_server \
   --model-path "$CKPT" \
   --trust-remote-code \
-  --tp-size 8 \
+  --tp-size "${GLM5V_TP_SIZE:-8}" \
   $QUANT_ARGS \
   --attention-backend "${GLM5V_ATTN_BACKEND:-dsa}" \
   --mm-attention-backend sdpa \