| { |
| "bits": 4, |
| "act_bits": 4, |
| "data_type": "nv_fp", |
| "act_data_type": "nv_fp4_with_static_gs", |
| "group_size": 16, |
| "act_group_size": 16, |
| "sym": true, |
| "act_sym": true, |
| "act_dynamic": true, |
| "low_gpu_mem_usage": true, |
| "seqlen": 1024, |
| "autoround_version": "0.13.1", |
| "block_name_to_quantize": "model.layers", |
| "quant_method": "auto-round", |
| "packing_format": "auto_round:llm_compressor", |
| "extra_config": { |
| "model.layers.0.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| ".*mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| ".*self_attn.*": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| ".*model\\.layers\\.[0-9]\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| } |
| } |
| } |
