File size: 2,453 Bytes
ff32ca7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """Compression recipe used to create this checkpoint.
Run quantization from the original BF16 checkpoint, not from an already
quantized checkpoint.
AWQ needs calibration data. This build used 128 samples from
`lmms-lab/flickr30k`, each rendered at 448x448 (a single InternVL patch) with
generic English/Korean scene-description prompts. Re-run with in-domain images
if your target distribution is far from everyday photography.
Targets are regex-scoped to the language decoder, so the vision tower,
multimodal projector, input embeddings, and lm_head are never matched and stay
in BF16. `ignore` is therefore empty by design.
"""
from llmcompressor.modifiers.awq import AWQModifier
from llmcompressor.modifiers.quantization import QuantizationModifier
ATTENTION = r"re:^model\.language_model\.layers\.\d+\.self_attn\.(q_proj|k_proj|v_proj|o_proj)$"
MLP = r"re:^model\.language_model\.layers\.\d+\.mlp\.(gate_proj|up_proj|down_proj)$"
RECIPE = [
AWQModifier(
mappings=[
{
"smooth_layer": r"re:^model\.language_model\.layers\.\d+\.input_layernorm$",
"balance_layers": [
r"re:^model\.language_model\.layers\.\d+\.self_attn\.q_proj$",
r"re:^model\.language_model\.layers\.\d+\.self_attn\.k_proj$",
r"re:^model\.language_model\.layers\.\d+\.self_attn\.v_proj$",
],
},
{
"smooth_layer": r"re:^model\.language_model\.layers\.\d+\.self_attn\.v_proj$",
"balance_layers": [
r"re:^model\.language_model\.layers\.\d+\.self_attn\.o_proj$",
],
},
{
"smooth_layer": r"re:^model\.language_model\.layers\.\d+\.post_attention_layernorm$",
"balance_layers": [
r"re:^model\.language_model\.layers\.\d+\.mlp\.gate_proj$",
r"re:^model\.language_model\.layers\.\d+\.mlp\.up_proj$",
],
},
{
"smooth_layer": r"re:^model\.language_model\.layers\.\d+\.mlp\.up_proj$",
"balance_layers": [
r"re:^model\.language_model\.layers\.\d+\.mlp\.down_proj$",
],
},
],
duo_scaling=True,
n_grid=20,
),
QuantizationModifier(
targets=[ATTENTION, MLP],
ignore=[],
scheme="W4A16_ASYM",
),
]
|