Hugging Face's logo

yasu-oh
/
gemma-4-E2B-it-qat-NVFP4

Image-Text-to-Text
Transformers
Safetensors
gemma4
quantization
nvfp4
conversational
compressed-tensors
Model card Files
xet
 Community
yasu-oh commited on
Commit
da0e72a
·
1 Parent(s): 1ccb861

v2

Browse files
Files changed (5) hide show
  1. README.md +39 -4
  2. config.json +77 -4
  3. model.safetensors +3 -0
  4. recipe.yaml +37 -3
  5. tokenizer.json +3 -0
README.md CHANGED
@@ -46,10 +46,40 @@ The calibration dataset consists of:
46
  ```yaml
47
  default_stage:
48
  default_modifiers:
49
- QuantizationModifier:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  targets: [Linear]
51
- ignore: ['re:.*vision.*', 're:.*audio.*', lm_head, 're:.*embed.*', 're:.*router.*']
52
- scheme: NVFP4
53
  kv_cache_scheme:
54
  num_bits: 8
55
  type: float
@@ -65,6 +95,10 @@ default_stage:
65
  observer_kwargs: {}
66
  bypass_divisibility_checks: false
67
  requires_calibration_data: true
 
 
 
 
68
  ```
69
 
70
  ## Reproducibility
@@ -73,6 +107,7 @@ The quantization run used:
73
 
74
  ```text
75
  transformers: 5.12.1
76
- llmcompressor: 0.12.1.dev89+g9e1fcd014
 
77
  ```
78
 
 
46
  ```yaml
47
  default_stage:
48
  default_modifiers:
49
+ GPTQModifier:
50
+ config_groups:
51
+ group_0:
52
+ targets: [Linear]
53
+ weights:
54
+ num_bits: 4
55
+ type: float
56
+ symmetric: true
57
+ group_size: 16
58
+ strategy: tensor_group
59
+ block_structure: null
60
+ dynamic: false
61
+ actorder: static
62
+ scale_dtype: torch.float8_e4m3fn
63
+ zp_dtype: null
64
+ observer: imatrix_mse
65
+ observer_kwargs: {strict: true}
66
+ input_activations:
67
+ num_bits: 4
68
+ type: float
69
+ symmetric: true
70
+ group_size: 16
71
+ strategy: tensor_group
72
+ block_structure: null
73
+ dynamic: local
74
+ actorder: null
75
+ scale_dtype: torch.float8_e4m3fn
76
+ zp_dtype: null
77
+ observer: static_minmax
78
+ observer_kwargs: {}
79
+ output_activations: null
80
+ format: null
81
  targets: [Linear]
82
+ ignore: [lm_head, 're:.*embed.*', 're:.*vision.*', 're:.*audio.*', 're:.*router.*', 're:.*per_layer.*']
 
83
  kv_cache_scheme:
84
  num_bits: 8
85
  type: float
 
95
  observer_kwargs: {}
96
  bypass_divisibility_checks: false
97
  requires_calibration_data: true
98
+ block_size: 128
99
+ dampening_frac: 0.01
100
+ actorder: static
101
+ offload_hessians: false
102
  ```
103
 
104
  ## Reproducibility
 
107
 
108
  ```text
109
  transformers: 5.12.1
110
+ llmcompressor: 0.12.1.dev92+g8cec0acc1
111
+ compressed-tensors: 0.17.2a20260729
112
  ```
113
 
config.json CHANGED
@@ -79,13 +79,15 @@
79
  "Linear"
80
  ],
81
  "weights": {
82
- "actorder": null,
83
  "block_structure": null,
84
  "dynamic": false,
85
  "group_size": 16,
86
  "num_bits": 4,
87
- "observer": "memoryless_minmax",
88
- "observer_kwargs": {},
 
 
89
  "scale_dtype": "torch.float8_e4m3fn",
90
  "strategy": "tensor_group",
91
  "symmetric": true,
@@ -210,6 +212,77 @@
210
  "model.vision_tower.encoder.layers.15.mlp.gate_proj.linear",
211
  "model.vision_tower.encoder.layers.15.mlp.up_proj.linear",
212
  "model.vision_tower.encoder.layers.15.mlp.down_proj.linear",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
213
  "model.audio_tower.subsample_conv_projection.input_proj_linear",
214
  "model.audio_tower.layers.0.feed_forward1.ffw_layer_1.linear",
215
  "model.audio_tower.layers.0.feed_forward1.ffw_layer_2.linear",
@@ -366,7 +439,7 @@
366
  "quantization_status": "compressed",
367
  "sparsity_config": {},
368
  "transform_config": {},
369
- "version": "0.1.dev549+g50c1143"
370
  },
371
  "text_config": {
372
  "attention_bias": false,
 
79
  "Linear"
80
  ],
81
  "weights": {
82
+ "actorder": "static",
83
  "block_structure": null,
84
  "dynamic": false,
85
  "group_size": 16,
86
  "num_bits": 4,
87
+ "observer": "imatrix_mse",
88
+ "observer_kwargs": {
89
+ "strict": true
90
+ },
91
  "scale_dtype": "torch.float8_e4m3fn",
92
  "strategy": "tensor_group",
93
  "symmetric": true,
 
212
  "model.vision_tower.encoder.layers.15.mlp.gate_proj.linear",
213
  "model.vision_tower.encoder.layers.15.mlp.up_proj.linear",
214
  "model.vision_tower.encoder.layers.15.mlp.down_proj.linear",
215
+ "model.language_model.layers.0.per_layer_input_gate",
216
+ "model.language_model.layers.0.per_layer_projection",
217
+ "model.language_model.layers.1.per_layer_input_gate",
218
+ "model.language_model.layers.1.per_layer_projection",
219
+ "model.language_model.layers.2.per_layer_input_gate",
220
+ "model.language_model.layers.2.per_layer_projection",
221
+ "model.language_model.layers.3.per_layer_input_gate",
222
+ "model.language_model.layers.3.per_layer_projection",
223
+ "model.language_model.layers.4.per_layer_input_gate",
224
+ "model.language_model.layers.4.per_layer_projection",
225
+ "model.language_model.layers.5.per_layer_input_gate",
226
+ "model.language_model.layers.5.per_layer_projection",
227
+ "model.language_model.layers.6.per_layer_input_gate",
228
+ "model.language_model.layers.6.per_layer_projection",
229
+ "model.language_model.layers.7.per_layer_input_gate",
230
+ "model.language_model.layers.7.per_layer_projection",
231
+ "model.language_model.layers.8.per_layer_input_gate",
232
+ "model.language_model.layers.8.per_layer_projection",
233
+ "model.language_model.layers.9.per_layer_input_gate",
234
+ "model.language_model.layers.9.per_layer_projection",
235
+ "model.language_model.layers.10.per_layer_input_gate",
236
+ "model.language_model.layers.10.per_layer_projection",
237
+ "model.language_model.layers.11.per_layer_input_gate",
238
+ "model.language_model.layers.11.per_layer_projection",
239
+ "model.language_model.layers.12.per_layer_input_gate",
240
+ "model.language_model.layers.12.per_layer_projection",
241
+ "model.language_model.layers.13.per_layer_input_gate",
242
+ "model.language_model.layers.13.per_layer_projection",
243
+ "model.language_model.layers.14.per_layer_input_gate",
244
+ "model.language_model.layers.14.per_layer_projection",
245
+ "model.language_model.layers.15.per_layer_input_gate",
246
+ "model.language_model.layers.15.per_layer_projection",
247
+ "model.language_model.layers.16.per_layer_input_gate",
248
+ "model.language_model.layers.16.per_layer_projection",
249
+ "model.language_model.layers.17.per_layer_input_gate",
250
+ "model.language_model.layers.17.per_layer_projection",
251
+ "model.language_model.layers.18.per_layer_input_gate",
252
+ "model.language_model.layers.18.per_layer_projection",
253
+ "model.language_model.layers.19.per_layer_input_gate",
254
+ "model.language_model.layers.19.per_layer_projection",
255
+ "model.language_model.layers.20.per_layer_input_gate",
256
+ "model.language_model.layers.20.per_layer_projection",
257
+ "model.language_model.layers.21.per_layer_input_gate",
258
+ "model.language_model.layers.21.per_layer_projection",
259
+ "model.language_model.layers.22.per_layer_input_gate",
260
+ "model.language_model.layers.22.per_layer_projection",
261
+ "model.language_model.layers.23.per_layer_input_gate",
262
+ "model.language_model.layers.23.per_layer_projection",
263
+ "model.language_model.layers.24.per_layer_input_gate",
264
+ "model.language_model.layers.24.per_layer_projection",
265
+ "model.language_model.layers.25.per_layer_input_gate",
266
+ "model.language_model.layers.25.per_layer_projection",
267
+ "model.language_model.layers.26.per_layer_input_gate",
268
+ "model.language_model.layers.26.per_layer_projection",
269
+ "model.language_model.layers.27.per_layer_input_gate",
270
+ "model.language_model.layers.27.per_layer_projection",
271
+ "model.language_model.layers.28.per_layer_input_gate",
272
+ "model.language_model.layers.28.per_layer_projection",
273
+ "model.language_model.layers.29.per_layer_input_gate",
274
+ "model.language_model.layers.29.per_layer_projection",
275
+ "model.language_model.layers.30.per_layer_input_gate",
276
+ "model.language_model.layers.30.per_layer_projection",
277
+ "model.language_model.layers.31.per_layer_input_gate",
278
+ "model.language_model.layers.31.per_layer_projection",
279
+ "model.language_model.layers.32.per_layer_input_gate",
280
+ "model.language_model.layers.32.per_layer_projection",
281
+ "model.language_model.layers.33.per_layer_input_gate",
282
+ "model.language_model.layers.33.per_layer_projection",
283
+ "model.language_model.layers.34.per_layer_input_gate",
284
+ "model.language_model.layers.34.per_layer_projection",
285
+ "model.language_model.per_layer_model_projection",
286
  "model.audio_tower.subsample_conv_projection.input_proj_linear",
287
  "model.audio_tower.layers.0.feed_forward1.ffw_layer_1.linear",
288
  "model.audio_tower.layers.0.feed_forward1.ffw_layer_2.linear",
 
439
  "quantization_status": "compressed",
440
  "sparsity_config": {},
441
  "transform_config": {},
442
+ "version": "0.17.2.a20260729"
443
  },
444
  "text_config": {
445
  "attention_bias": false,
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce2fb540c9dd3d74d843a3afa82e5b80c24aa8d2e25bb5ed4ae4e0c64b35c7a3
3
+ size 7570358490
recipe.yaml CHANGED
@@ -1,9 +1,39 @@
1
  default_stage:
2
  default_modifiers:
3
- QuantizationModifier:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  targets: [Linear]
5
- ignore: ['re:.*vision.*', 're:.*audio.*', lm_head, 're:.*embed.*', 're:.*router.*']
6
- scheme: NVFP4
7
  kv_cache_scheme:
8
  num_bits: 8
9
  type: float
@@ -19,3 +49,7 @@ default_stage:
19
  observer_kwargs: {}
20
  bypass_divisibility_checks: false
21
  requires_calibration_data: true
 
 
 
 
 
1
  default_stage:
2
  default_modifiers:
3
+ GPTQModifier:
4
+ config_groups:
5
+ group_0:
6
+ targets: [Linear]
7
+ weights:
8
+ num_bits: 4
9
+ type: float
10
+ symmetric: true
11
+ group_size: 16
12
+ strategy: tensor_group
13
+ block_structure: null
14
+ dynamic: false
15
+ actorder: static
16
+ scale_dtype: torch.float8_e4m3fn
17
+ zp_dtype: null
18
+ observer: imatrix_mse
19
+ observer_kwargs: {strict: true}
20
+ input_activations:
21
+ num_bits: 4
22
+ type: float
23
+ symmetric: true
24
+ group_size: 16
25
+ strategy: tensor_group
26
+ block_structure: null
27
+ dynamic: local
28
+ actorder: null
29
+ scale_dtype: torch.float8_e4m3fn
30
+ zp_dtype: null
31
+ observer: static_minmax
32
+ observer_kwargs: {}
33
+ output_activations: null
34
+ format: null
35
  targets: [Linear]
36
+ ignore: [lm_head, 're:.*embed.*', 're:.*vision.*', 're:.*audio.*', 're:.*router.*', 're:.*per_layer.*']
 
37
  kv_cache_scheme:
38
  num_bits: 8
39
  type: float
 
49
  observer_kwargs: {}
50
  bypass_divisibility_checks: false
51
  requires_calibration_data: true
52
+ block_size: 128
53
+ dampening_frac: 0.01
54
+ actorder: static
55
+ offload_hessians: false
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
3
+ size 32169626