File size: 7,488 Bytes
3755c6c 14aa7c0 3755c6c 14aa7c0 3755c6c 14aa7c0 3755c6c 14aa7c0 3755c6c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | # SPDX-License-Identifier: Apache-2.0
"""Glm5vConfig — remote-code config carried inside the assembled GLM5V SGLang
checkpoint (referenced by config.json ``auto_map``; loaded with
``--trust-remote-code``, which the checkpoint already requires for the Kimi
image-processor remote code).
Self-contained: depends only on ``transformers``. Mirrors SGLang's in-tree
``KimiK25Config`` structure (``vision_config`` + ``text_config`` + media
placeholder fields) with GLM-5.2 as the text model:
* ``text_config`` -> ``GlmMoeDsaConfig`` (transformers-native ``glm_moe_dsa``).
* ``vision_config``-> MoonViT fields; ``text_hidden_size`` (projector output
dim) retargeted to GLM hidden 6144.
* ``media_placeholder_token_id`` -> GLM ``<|image|>`` = 154854.
"""
from transformers import AutoConfig
from transformers.configuration_utils import PretrainedConfig
class Glm5vVisionConfig(PretrainedConfig):
"""MoonViT vision tower + PatchMerger projector config.
Field names/defaults mirror SGLang's ``KimiK25VisionConfig`` (declared
names like ``hidden_size``) while the official Kimi checkpoint's ``vt_*``
names arrive via **kwargs and are stored as attributes — SGLang's model
code reads both families (tower: ``hidden_size``; projector:
``vt_hidden_size``/``text_hidden_size``).
"""
model_type = "glm5v_vision"
def __init__(
self,
# Vision tower
patch_size: int = 14,
init_pos_emb_height: int = 64,
init_pos_emb_width: int = 64,
init_pos_emb_time: int = 4,
pos_emb_type: str = "divided_fixed",
num_attention_heads: int = 16,
num_hidden_layers: int = 27,
hidden_size: int = 1152,
intermediate_size: int = 4304,
merge_kernel_size=(2, 2),
video_attn_type: str = "spatial_temporal",
merge_type: str = "sd2_tpool",
# MM projector
mm_projector_type: str = "patchmerger",
mm_hidden_size: int | None = None,
vt_hidden_size: int | None = None, # SGLang kimi_k25 projector reads this (== vision-tower hidden)
projector_hidden_act: str = "gelu",
projector_ln_eps: float = 1e-5,
text_hidden_size: int = 6144, # GLM-5.2 hidden (Kimi default is 7168)
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.init_pos_emb_height = init_pos_emb_height
self.init_pos_emb_width = init_pos_emb_width
self.init_pos_emb_time = init_pos_emb_time
self.pos_emb_type = pos_emb_type
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.merge_kernel_size = merge_kernel_size
self.video_attn_type = video_attn_type
self.merge_type = merge_type
self.mm_projector_type = mm_projector_type
self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else hidden_size
self.vt_hidden_size = vt_hidden_size if vt_hidden_size is not None else hidden_size
self.projector_hidden_act = projector_hidden_act
self.projector_ln_eps = projector_ln_eps
self.text_hidden_size = text_hidden_size
def __getattr__(self, name):
# SGLang's kimi_k25 reads vt_-prefixed vision fields (vt_hidden_size, vt_intermediate_size, ...)
# that our config declares without the prefix; alias any missing vt_* to the base attribute.
# Reads __dict__ directly (no recursion) and raises normally if the base isn't set.
if name.startswith("vt_"):
d = object.__getattribute__(self, "__dict__")
base = name[3:]
if base in d:
return d[base]
raise AttributeError(name)
class Glm5vConfig(PretrainedConfig):
"""glm5v top-level config: MoonViT ``vision_config`` + GLM-5.2 ``text_config``."""
model_type = "glm5v"
def __init__(
self,
text_config=None,
vision_config=None,
ignore_index: int = -100,
media_placeholder_token_id: int = 154854, # GLM <|image|>
pad_token_id: int = 154820,
use_unified_vision_chunk: bool = True,
video_placeholder: str = "<|glm5v_video_placeholder|>",
encoder_only: bool = False,
language_only: bool = False,
**kwargs,
):
# Vision config (MoonViT).
if vision_config is None:
self.vision_config = Glm5vVisionConfig()
elif isinstance(vision_config, dict):
self.vision_config = Glm5vVisionConfig(**vision_config)
else:
self.vision_config = vision_config
# Text config (GLM-5.2 / glm_moe_dsa), built via AutoConfig so the
# transformers-native GlmMoeDsaConfig class is used.
raw_text = dict(text_config) if isinstance(text_config, dict) else None
if text_config is None:
self.text_config = AutoConfig.for_model("glm_moe_dsa")
elif isinstance(text_config, dict):
tc = dict(text_config)
tc.setdefault("model_type", "glm_moe_dsa")
# Newer transformers (in the SGLang serving image) validates `layer_types`
# via a StrictDataclass and rejects the legacy DSA value
# "deepseek_sparse_attention". The DSA attention path is selected from
# model_type + the DSA config fields (index_topk etc.), NOT from layer_types,
# so drop it to pass validation without changing behavior.
tc.pop("layer_types", None)
self.text_config = AutoConfig.for_model(**tc)
else:
self.text_config = text_config
# transformers 5.8.x GlmMoeDsaConfig drops/clobbers raw DSA fields the
# sparse-attention path needs. SGLang applies this same restore for
# bare GlmMoeDsaForCausalLM checkpoints (see its HfModelConfigParser;
# fixed upstream by transformers PR #46338, gone once >= 5.10); our
# top-level arch is Glm5v so we replicate it here.
if raw_text is not None:
for key in ("qk_rope_head_dim", "index_topk_freq"):
if key in raw_text:
setattr(self.text_config, key, raw_text[key])
if hasattr(self.text_config, "qk_nope_head_dim") and hasattr(
self.text_config, "qk_rope_head_dim"
):
self.text_config.qk_head_dim = (
self.text_config.qk_nope_head_dim
+ self.text_config.qk_rope_head_dim
)
self.ignore_index = ignore_index
self.media_placeholder_token_id = media_placeholder_token_id
self.use_unified_vision_chunk = use_unified_vision_chunk
self.video_placeholder = video_placeholder
self.encoder_only = encoder_only
self.language_only = language_only
# Propagate quantization config from the text model (Kimi pattern):
# only the GLM text Linears are FP8; vision/projector stay bf16 by
# construction in the model code.
if getattr(self.text_config, "quantization_config", None) is not None:
self.quantization_config = self.text_config.quantization_config
super().__init__(pad_token_id=pad_token_id, **kwargs)
@property
def hidden_size(self) -> int:
return self.text_config.hidden_size
@property
def vocab_size(self) -> int:
return self.text_config.vocab_size
|