# GLM-5.2-MXFP8-NVFP4-NF3-Hybrid-Vision — exact measured serving configuration.
# 4x 96GB sm120 GPUs, PCIe host. Serves GLM-5.2-Vision on :8000 with a
# 509,467-token FP8 KV pool @ 250k max context (DCP4, util 0.974).
#   hf download 0xSero/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid-Vision --local-dir ./glm52-vision
#   MODEL_DIR=./glm52-vision docker compose up
services:
  glm52-vision:
    image: ghcr.io/0xsero/glm52-nf3-vision:v3-attn
    container_name: glm52-vision
    network_mode: host
    ipc: host
    shm_size: 32g
    init: true
    gpus: all
    ulimits:
      memlock: -1
      stack: 67108864
      nofile: 1048576
    environment:
      - CUDA_VISIBLE_DEVICES=0,1,2,3
      - CUDA_DEVICE_ORDER=PCI_BUS_ID
      - CUDA_DEVICE_MAX_CONNECTIONS=32
      - CUTE_DSL_ARCH=sm_120a
      - OMP_NUM_THREADS=16
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
      - SAFETENSORS_FAST_GPU=1
      - NCCL_IB_DISABLE=1
      - NCCL_P2P_LEVEL=SYS
      - NCCL_PROTO=LL,LL128,Simple
      # b12x feature set (NF3 kernel, sparse indexer, PCIe allreduce)
      - VLLM_USE_B12X_FP8_GEMM=1
      - VLLM_USE_B12X_MOE=1
      - VLLM_USE_B12X_SPARSE_INDEXER=1
      - VLLM_USE_V2_MODEL_RUNNER=1
      - B12X_W4A16_TC_DECODE=1
      - B12X_MOE_FORCE_A16=1
      - B12X_DENSE_SPLITK_TURBO=1
      - VLLM_DCP_GLOBAL_TOPK=1
      - VLLM_DCP_SHARD_DRAFT=1
      - VLLM_ENABLE_PCIE_ALLREDUCE=1
      - VLLM_PCIE_ALLREDUCE_BACKEND=b12x
      - VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE=64KB
      - VLLM_PCIE_ONESHOT_MAX_BYTES=65536
      - VLLM_CPP_AR_1STAGE_NCCL_CUTOFF=56KB
      - VLLM_CPP_AR_IGNORE_CUTOFF_MAX_ROWS=0
      - VLLM_RTX6K_FUSED_ALLREDUCE_ADD=0
      - VLLM_RTX6K_FUSED_ALLREDUCE_ADD_END_BARRIER=0
      - VLLM_DISABLE_SHARED_EXPERTS_STREAM=0
      - VLLM_MEMORY_PROFILE_INCLUDE_ATTN=1
      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
      - VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel
      # hybrid NF3/NVFP4/mxfp8 loader
      - HYBRID_TIER=both
      - HYBRID_KEPT=b12x_nf3
      - HYBRID_NF3=b12x_nf3
      - HYBRID_B12X_MAX_TOKENS=2048
      - HYBRID_MXFP8_NATIVE=1
      # JIT cache
      - XDG_CACHE_HOME=/cache/jit
      - VLLM_CACHE_DIR=/cache/jit/vllm
      - TRITON_CACHE_DIR=/cache/jit/triton
    volumes:
      - ${MODEL_DIR:?set MODEL_DIR to the downloaded checkpoint dir}:/model:ro
      # measured overlay: kv_b_proj + shared_experts stay true BF16
      - ${MODEL_DIR:?}/mxfp8_tier_nokvb.json:/opt/venv/lib/python3.12/site-packages/mxfp8_tier.json:ro
      - glm52-vision-jit:/cache
    command:
      - vllm
      - serve
      - /model
      - --served-model-name=GLM-5.2-Vision
      - --host=0.0.0.0
      - --port=8000
      - --trust-remote-code
      - --tensor-parallel-size=4
      - --decode-context-parallel-size=4
      - --dcp-comm-backend=ag_rs
      - --dcp-kv-cache-interleave-size=1
      - --kv-cache-dtype=fp8
      - --attention-backend=B12X_MLA_SPARSE
      - --moe-backend=b12x
      - --load-format=fastsafetensors
      - -cc.pass_config.fuse_allreduce_rms=True
      - --gpu-memory-utilization=0.974
      - --max-model-len=250000
      - --max-num-seqs=4
      - --max-num-batched-tokens=2048
      - --max-cudagraph-capture-size=64
      - --async-scheduling
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --enable-auto-tool-choice
      - --tool-call-parser=glm47
      - --reasoning-parser=glm45
      - '--default-chat-template-kwargs={"reasoning_effort":"high"}'
      - '--hf-overrides={"use_index_cache":true,"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'
volumes:
  glm52-vision-jit: