Upload FP8-quantized model (FP8_DYNAMIC, via llm-compressor)

README.md

@@ -41,7 +41,7 @@ language:
 | **Weight Quantization** | Symmetric, static, per-channel → FP8 (E4M3) |
 | **Activation Quantization** | Symmetric, dynamic, per-token → FP8 (E4M3) |
 | **Format** | `compressed-tensors` (vLLM-native) |
-| **Quantized Size** | ~4.91 GB |
+| **Quantized Size** | ~5.43 GB |
 | **Tool** | [`llm-compressor`](https://github.com/vllm-project/llm-compressor) |
 | **Date** | 2026-04-16 |
 

config.json

@@ -72,6 +72,231 @@
     "format": "float-quantized",
     "global_compression_ratio": null,
     "ignore": [
+      "audio_tower.layers.0.self_attn.q_proj",
+      "audio_tower.layers.0.self_attn.k_proj",
+      "audio_tower.layers.0.self_attn.v_proj",
+      "audio_tower.layers.0.self_attn.o_proj",
+      "audio_tower.layers.0.mlp.gate_proj",
+      "audio_tower.layers.0.mlp.up_proj",
+      "audio_tower.layers.0.mlp.down_proj",
+      "audio_tower.layers.1.self_attn.q_proj",
+      "audio_tower.layers.1.self_attn.k_proj",
+      "audio_tower.layers.1.self_attn.v_proj",
+      "audio_tower.layers.1.self_attn.o_proj",
+      "audio_tower.layers.1.mlp.gate_proj",
+      "audio_tower.layers.1.mlp.up_proj",
+      "audio_tower.layers.1.mlp.down_proj",
+      "audio_tower.layers.2.self_attn.q_proj",
+      "audio_tower.layers.2.self_attn.k_proj",
+      "audio_tower.layers.2.self_attn.v_proj",
+      "audio_tower.layers.2.self_attn.o_proj",
+      "audio_tower.layers.2.mlp.gate_proj",
+      "audio_tower.layers.2.mlp.up_proj",
+      "audio_tower.layers.2.mlp.down_proj",
+      "audio_tower.layers.3.self_attn.q_proj",
+      "audio_tower.layers.3.self_attn.k_proj",
+      "audio_tower.layers.3.self_attn.v_proj",
+      "audio_tower.layers.3.self_attn.o_proj",
+      "audio_tower.layers.3.mlp.gate_proj",
+      "audio_tower.layers.3.mlp.up_proj",
+      "audio_tower.layers.3.mlp.down_proj",
+      "audio_tower.layers.4.self_attn.q_proj",
+      "audio_tower.layers.4.self_attn.k_proj",
+      "audio_tower.layers.4.self_attn.v_proj",
+      "audio_tower.layers.4.self_attn.o_proj",
+      "audio_tower.layers.4.mlp.gate_proj",
+      "audio_tower.layers.4.mlp.up_proj",
+      "audio_tower.layers.4.mlp.down_proj",
+      "audio_tower.layers.5.self_attn.q_proj",
+      "audio_tower.layers.5.self_attn.k_proj",
+      "audio_tower.layers.5.self_attn.v_proj",
+      "audio_tower.layers.5.self_attn.o_proj",
+      "audio_tower.layers.5.mlp.gate_proj",
+      "audio_tower.layers.5.mlp.up_proj",
+      "audio_tower.layers.5.mlp.down_proj",
+      "audio_tower.layers.6.self_attn.q_proj",
+      "audio_tower.layers.6.self_attn.k_proj",
+      "audio_tower.layers.6.self_attn.v_proj",
+      "audio_tower.layers.6.self_attn.o_proj",
+      "audio_tower.layers.6.mlp.gate_proj",
+      "audio_tower.layers.6.mlp.up_proj",
+      "audio_tower.layers.6.mlp.down_proj",
+      "audio_tower.layers.7.self_attn.q_proj",
+      "audio_tower.layers.7.self_attn.k_proj",
+      "audio_tower.layers.7.self_attn.v_proj",
+      "audio_tower.layers.7.self_attn.o_proj",
+      "audio_tower.layers.7.mlp.gate_proj",
+      "audio_tower.layers.7.mlp.up_proj",
+      "audio_tower.layers.7.mlp.down_proj",
+      "audio_tower.layers.8.self_attn.q_proj",
+      "audio_tower.layers.8.self_attn.k_proj",
+      "audio_tower.layers.8.self_attn.v_proj",
+      "audio_tower.layers.8.self_attn.o_proj",
+      "audio_tower.layers.8.mlp.gate_proj",
+      "audio_tower.layers.8.mlp.up_proj",
+      "audio_tower.layers.8.mlp.down_proj",
+      "audio_tower.layers.9.self_attn.q_proj",
+      "audio_tower.layers.9.self_attn.k_proj",
+      "audio_tower.layers.9.self_attn.v_proj",
+      "audio_tower.layers.9.self_attn.o_proj",
+      "audio_tower.layers.9.mlp.gate_proj",
+      "audio_tower.layers.9.mlp.up_proj",
+      "audio_tower.layers.9.mlp.down_proj",
+      "audio_tower.layers.10.self_attn.q_proj",
+      "audio_tower.layers.10.self_attn.k_proj",
+      "audio_tower.layers.10.self_attn.v_proj",
+      "audio_tower.layers.10.self_attn.o_proj",
+      "audio_tower.layers.10.mlp.gate_proj",
+      "audio_tower.layers.10.mlp.up_proj",
+      "audio_tower.layers.10.mlp.down_proj",
+      "audio_tower.layers.11.self_attn.q_proj",
+      "audio_tower.layers.11.self_attn.k_proj",
+      "audio_tower.layers.11.self_attn.v_proj",
+      "audio_tower.layers.11.self_attn.o_proj",
+      "audio_tower.layers.11.mlp.gate_proj",
+      "audio_tower.layers.11.mlp.up_proj",
+      "audio_tower.layers.11.mlp.down_proj",
+      "audio_tower.layers.12.self_attn.q_proj",
+      "audio_tower.layers.12.self_attn.k_proj",
+      "audio_tower.layers.12.self_attn.v_proj",
+      "audio_tower.layers.12.self_attn.o_proj",
+      "audio_tower.layers.12.mlp.gate_proj",
+      "audio_tower.layers.12.mlp.up_proj",
+      "audio_tower.layers.12.mlp.down_proj",
+      "audio_tower.layers.13.self_attn.q_proj",
+      "audio_tower.layers.13.self_attn.k_proj",
+      "audio_tower.layers.13.self_attn.v_proj",
+      "audio_tower.layers.13.self_attn.o_proj",
+      "audio_tower.layers.13.mlp.gate_proj",
+      "audio_tower.layers.13.mlp.up_proj",
+      "audio_tower.layers.13.mlp.down_proj",
+      "audio_tower.layers.14.self_attn.q_proj",
+      "audio_tower.layers.14.self_attn.k_proj",
+      "audio_tower.layers.14.self_attn.v_proj",
+      "audio_tower.layers.14.self_attn.o_proj",
+      "audio_tower.layers.14.mlp.gate_proj",
+      "audio_tower.layers.14.mlp.up_proj",
+      "audio_tower.layers.14.mlp.down_proj",
+      "audio_tower.layers.15.self_attn.q_proj",
+      "audio_tower.layers.15.self_attn.k_proj",
+      "audio_tower.layers.15.self_attn.v_proj",
+      "audio_tower.layers.15.self_attn.o_proj",
+      "audio_tower.layers.15.mlp.gate_proj",
+      "audio_tower.layers.15.mlp.up_proj",
+      "audio_tower.layers.15.mlp.down_proj",
+      "audio_tower.layers.16.self_attn.q_proj",
+      "audio_tower.layers.16.self_attn.k_proj",
+      "audio_tower.layers.16.self_attn.v_proj",
+      "audio_tower.layers.16.self_attn.o_proj",
+      "audio_tower.layers.16.mlp.gate_proj",
+      "audio_tower.layers.16.mlp.up_proj",
+      "audio_tower.layers.16.mlp.down_proj",
+      "audio_tower.layers.17.self_attn.q_proj",
+      "audio_tower.layers.17.self_attn.k_proj",
+      "audio_tower.layers.17.self_attn.v_proj",
+      "audio_tower.layers.17.self_attn.o_proj",
+      "audio_tower.layers.17.mlp.gate_proj",
+      "audio_tower.layers.17.mlp.up_proj",
+      "audio_tower.layers.17.mlp.down_proj",
+      "audio_tower.layers.18.self_attn.q_proj",
+      "audio_tower.layers.18.self_attn.k_proj",
+      "audio_tower.layers.18.self_attn.v_proj",
+      "audio_tower.layers.18.self_attn.o_proj",
+      "audio_tower.layers.18.mlp.gate_proj",
+      "audio_tower.layers.18.mlp.up_proj",
+      "audio_tower.layers.18.mlp.down_proj",
+      "audio_tower.layers.19.self_attn.q_proj",
+      "audio_tower.layers.19.self_attn.k_proj",
+      "audio_tower.layers.19.self_attn.v_proj",
+      "audio_tower.layers.19.self_attn.o_proj",
+      "audio_tower.layers.19.mlp.gate_proj",
+      "audio_tower.layers.19.mlp.up_proj",
+      "audio_tower.layers.19.mlp.down_proj",
+      "audio_tower.layers.20.self_attn.q_proj",
+      "audio_tower.layers.20.self_attn.k_proj",
+      "audio_tower.layers.20.self_attn.v_proj",
+      "audio_tower.layers.20.self_attn.o_proj",
+      "audio_tower.layers.20.mlp.gate_proj",
+      "audio_tower.layers.20.mlp.up_proj",
+      "audio_tower.layers.20.mlp.down_proj",
+      "audio_tower.layers.21.self_attn.q_proj",
+      "audio_tower.layers.21.self_attn.k_proj",
+      "audio_tower.layers.21.self_attn.v_proj",
+      "audio_tower.layers.21.self_attn.o_proj",
+      "audio_tower.layers.21.mlp.gate_proj",
+      "audio_tower.layers.21.mlp.up_proj",
+      "audio_tower.layers.21.mlp.down_proj",
+      "audio_tower.layers.22.self_attn.q_proj",
+      "audio_tower.layers.22.self_attn.k_proj",
+      "audio_tower.layers.22.self_attn.v_proj",
+      "audio_tower.layers.22.self_attn.o_proj",
+      "audio_tower.layers.22.mlp.gate_proj",
+      "audio_tower.layers.22.mlp.up_proj",
+      "audio_tower.layers.22.mlp.down_proj",
+      "audio_tower.layers.23.self_attn.q_proj",
+      "audio_tower.layers.23.self_attn.k_proj",
+      "audio_tower.layers.23.self_attn.v_proj",
+      "audio_tower.layers.23.self_attn.o_proj",
+      "audio_tower.layers.23.mlp.gate_proj",
+      "audio_tower.layers.23.mlp.up_proj",
+      "audio_tower.layers.23.mlp.down_proj",
+      "audio_tower.layers.24.self_attn.q_proj",
+      "audio_tower.layers.24.self_attn.k_proj",
+      "audio_tower.layers.24.self_attn.v_proj",
+      "audio_tower.layers.24.self_attn.o_proj",
+      "audio_tower.layers.24.mlp.gate_proj",
+      "audio_tower.layers.24.mlp.up_proj",
+      "audio_tower.layers.24.mlp.down_proj",
+      "audio_tower.layers.25.self_attn.q_proj",
+      "audio_tower.layers.25.self_attn.k_proj",
+      "audio_tower.layers.25.self_attn.v_proj",
+      "audio_tower.layers.25.self_attn.o_proj",
+      "audio_tower.layers.25.mlp.gate_proj",
+      "audio_tower.layers.25.mlp.up_proj",
+      "audio_tower.layers.25.mlp.down_proj",
+      "audio_tower.layers.26.self_attn.q_proj",
+      "audio_tower.layers.26.self_attn.k_proj",
+      "audio_tower.layers.26.self_attn.v_proj",
+      "audio_tower.layers.26.self_attn.o_proj",
+      "audio_tower.layers.26.mlp.gate_proj",
+      "audio_tower.layers.26.mlp.up_proj",
+      "audio_tower.layers.26.mlp.down_proj",
+      "audio_tower.layers.27.self_attn.q_proj",
+      "audio_tower.layers.27.self_attn.k_proj",
+      "audio_tower.layers.27.self_attn.v_proj",
+      "audio_tower.layers.27.self_attn.o_proj",
+      "audio_tower.layers.27.mlp.gate_proj",
+      "audio_tower.layers.27.mlp.up_proj",
+      "audio_tower.layers.27.mlp.down_proj",
+      "audio_tower.layers.28.self_attn.q_proj",
+      "audio_tower.layers.28.self_attn.k_proj",
+      "audio_tower.layers.28.self_attn.v_proj",
+      "audio_tower.layers.28.self_attn.o_proj",
+      "audio_tower.layers.28.mlp.gate_proj",
+      "audio_tower.layers.28.mlp.up_proj",
+      "audio_tower.layers.28.mlp.down_proj",
+      "audio_tower.layers.29.self_attn.q_proj",
+      "audio_tower.layers.29.self_attn.k_proj",
+      "audio_tower.layers.29.self_attn.v_proj",
+      "audio_tower.layers.29.self_attn.o_proj",
+      "audio_tower.layers.29.mlp.gate_proj",
+      "audio_tower.layers.29.mlp.up_proj",
+      "audio_tower.layers.29.mlp.down_proj",
+      "audio_tower.layers.30.self_attn.q_proj",
+      "audio_tower.layers.30.self_attn.k_proj",
+      "audio_tower.layers.30.self_attn.v_proj",
+      "audio_tower.layers.30.self_attn.o_proj",
+      "audio_tower.layers.30.mlp.gate_proj",
+      "audio_tower.layers.30.mlp.up_proj",
+      "audio_tower.layers.30.mlp.down_proj",
+      "audio_tower.layers.31.self_attn.q_proj",
+      "audio_tower.layers.31.self_attn.k_proj",
+      "audio_tower.layers.31.self_attn.v_proj",
+      "audio_tower.layers.31.self_attn.o_proj",
+      "audio_tower.layers.31.mlp.gate_proj",
+      "audio_tower.layers.31.mlp.up_proj",
+      "audio_tower.layers.31.mlp.down_proj",
+      "language_model.lm_head",
       "multi_modal_projector.linear_1",
       "multi_modal_projector.linear_2"
     ],
@@ -80,7 +305,7 @@
     "quantization_status": "compressed",
     "sparsity_config": {},
     "transform_config": {},
-    "version": "0.15.1.dev9+g37f209a"
+    "version": "0.15.1.dev10+ge515fd2"
   },
   "text_config": {
     "attention_dropout": 0.0,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a32ebf406e269d9a15f0fd4e761b19d8dd119ce2eb5af76f8dad63e8fe622a15
-size 5269400752
+oid sha256:5afe289389dbd530fbc595280b27731fd7351e6f1665d89c060f98f79281a51d
+size 5829935856

recipe.yaml

@@ -2,6 +2,6 @@ default_stage:
   default_modifiers:
     QuantizationModifier:
       targets: [Linear]
-      ignore: [lm_head, 're:audio_encoder.*', 're:multi_modal_projector.*', 're:.*embed_tokens.*']
+      ignore: ['re:.*lm_head', 're:audio_tower.*', 're:multi_modal_projector.*']
       scheme: FP8_DYNAMIC
       bypass_divisibility_checks: false