everything-lane: rung docs + FP8-vision note
Browse files
serving/everything-lane/Dockerfile
CHANGED
|
@@ -5,13 +5,15 @@
|
|
| 5 |
# - kvq.py / kvq_kernels.py + the mla/sparse_mla/kv_cache/flashinfer files:
|
| 6 |
# compact KV rows for the sparse-MLA layers, gated by VLLM_KVQ_TILES.
|
| 7 |
# "8888" repacks the fp8_ds_mla row 656->528 B (bit-identical: the 128 B
|
| 8 |
-
# RoPE slot is provably inert -- qk_rope_head_dim=0); "
|
| 9 |
-
#
|
|
|
|
| 10 |
# Decode/prefill re-stage top-k rows into canonical 656 B staging consumed
|
| 11 |
# by the unmodified FlashInfer SM120 cubin. Unset = stock path, no change.
|
| 12 |
# - model.py / modelopt.py: the 512k lane's files plus quant_config
|
| 13 |
-
# passthrough for the embedding/vision construction sites (
|
| 14 |
-
# vision
|
|
|
|
| 15 |
ARG BASE=local/vllm-glm53:fp8attn-512k
|
| 16 |
FROM ${BASE}
|
| 17 |
ARG V=/usr/local/lib/python3.12/dist-packages/vllm
|
|
|
|
| 5 |
# - kvq.py / kvq_kernels.py + the mla/sparse_mla/kv_cache/flashinfer files:
|
| 6 |
# compact KV rows for the sparse-MLA layers, gated by VLLM_KVQ_TILES.
|
| 7 |
# "8888" repacks the fp8_ds_mla row 656->528 B (bit-identical: the 128 B
|
| 8 |
+
# RoPE slot is provably inert -- qk_rope_head_dim=0); "8666" (1 fp8 +
|
| 9 |
+
# 3 fp6 tiles, 444 B/row) is the shipped 512k+MTP rung (TF 95.57%);
|
| 10 |
+
# "6666" (all-fp6, 416 B/row) trades ~1.5 TF points for a thinner pin.
|
| 11 |
# Decode/prefill re-stage top-k rows into canonical 656 B staging consumed
|
| 12 |
# by the unmodified FlashInfer SM120 cubin. Unset = stock path, no change.
|
| 13 |
# - model.py / modelopt.py: the 512k lane's files plus quant_config
|
| 14 |
+
# passthrough for the embedding/vision construction sites (manifest-gated:
|
| 15 |
+
# the shipped checkpoint's block-FP8 vision tower loads through it;
|
| 16 |
+
# BF16-tower checkpoints construct bit-identically to the base image)
|
| 17 |
ARG BASE=local/vllm-glm53:fp8attn-512k
|
| 18 |
FROM ${BASE}
|
| 19 |
ARG V=/usr/local/lib/python3.12/dist-packages/vllm
|