| { |
| "schema_version": "ax.native_model.v1", |
| "model_family": "qwen3_5", |
| "tensor_format": "safetensors", |
| "layer_count": 40, |
| "hidden_size": 2048, |
| "intermediate_size": 0, |
| "attention_head_count": 16, |
| "attention_head_dim": 256, |
| "kv_head_count": 2, |
| "vocab_size": 248320, |
| "tie_word_embeddings": false, |
| "rope_theta": 10000000, |
| "no_rope_layer_interval": 0, |
| "intermediate_size_mlp": 0, |
| "attn_output_gate": true, |
| "partial_rotary_factor": 0.25, |
| "rms_norm_eps": 0.000001, |
| "moe_norm_topk_prob": true, |
| "hidden_size_per_layer_input": 0, |
| "linear_attention": { |
| "full_attention_interval": 4, |
| "num_value_heads": 32, |
| "num_key_heads": 16, |
| "key_head_dim": 128, |
| "value_head_dim": 128, |
| "conv_kernel_dim": 4 |
| }, |
| "moe": { |
| "expert_count": 256, |
| "experts_per_token": 8, |
| "expert_intermediate_size": 512, |
| "sigmoid_routing": false |
| }, |
| "tensors": [ |
| { |
| "name": "language_model.model.norm.weight", |
| "role": "final_norm", |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 15912883, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.lm_head.weight", |
| "role": "lm_head", |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 248320, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1156932019, |
| "length_bytes": 381419520 |
| }, |
| { |
| "name": "language_model.model.embed_tokens.weight", |
| "role": "token_embedding", |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 248320, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2437548452, |
| "length_bytes": 381419520 |
| }, |
| { |
| "name": "language_model.model.layers.0.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2026691364, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.0.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2852255268, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 923361124, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 574126628, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1562476580, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3292539172, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3291701284, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3295530020, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3293924388, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2860650276, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2842338532, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1480950692, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2818967972, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2842022628, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2841973476, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1981592228, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2835384804, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2852255012, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1718476132, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.1.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3539509892, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.1.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 539495140, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3757650308, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3539544196, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 887922916, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3961696644, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3294710820, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1481020836, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5121641700, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3295532068, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 539495076, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3539444356, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3539182148, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3539132996, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3526490212, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3526541412, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3519936612, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 887918308, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3513645156, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.2.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4203819716, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.2.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4203823812, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2104137828, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4203827908, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1565657380, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2077498212, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 808655044, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4653537220, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4642834372, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4405156804, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 887918244, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 775457828, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 519122628, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1441621764, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1481022884, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 785439780, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3976409476, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4194283908, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4194284164, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4666650884, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4682292132, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.3.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4682292644, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4667505156, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.3.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4692220068, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4653543684, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1491029572, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1942316644, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4910353572, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1163689828, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2065255876, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5120855268, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5120068836, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4653539332, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1574152900, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4700638372, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.4.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3983034756, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.4.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5146216868, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 239865924, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3983038852, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1551242532, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3975098756, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1551771428, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 238913604, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 238981188, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1732762980, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5139663204, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4634303428, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2004186436, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5132482596, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5139612004, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3962483076, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5132531812, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2842338276, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 5139663268, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.5.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 218195652, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.5.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 218191556, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1713339450, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 16848580, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 2076941156, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1677431802, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 983562554, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3725159482, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1227815802, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3081983780, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 896527140, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 4200837828, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 218724292, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 1978888804, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 3081408292, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 226330628, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 218990596, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 218724036, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00001-of-00006.safetensors", |
| "offset_bytes": 917065060, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.6.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2407268218, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.6.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175560378, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2416447482, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2147106426, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1705931834, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3725456442, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2835947514, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3471950394, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2348482426, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2617806842, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3471952442, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1678775290, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 990689978, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715867706, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3927569466, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2368962426, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1670091770, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2348482170, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1221524346, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1206795002, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2147105914, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.7.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030553274, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 984348986, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.7.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1670087674, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2356379514, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3253055546, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2407272314, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1005468410, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2850222138, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1237253050, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1220737914, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3506271034, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1713271866, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1439368186, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4383706810, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.8.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030481594, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.8.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175553978, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3270619706, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1238041594, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2398355322, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3495580346, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1704883258, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715861562, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2355593082, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3051728954, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 702355514, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1914682426, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 702355578, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 702357690, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3051614266, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 711106746, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 696064058, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 990689658, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4167456122, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.9.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715863610, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.9.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030623930, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3507057466, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1455096826, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715949626, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4165539130, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4163107898, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3496438522, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4166358330, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3726242874, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 990689594, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3496506170, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1220705018, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3714939706, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2348433018, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1207581434, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3708384058, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715255098, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 999111418, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.10.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2849660922, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.10.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1678840826, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4828303034, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4182380218, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4181855930, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3269833274, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1656423418, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175558074, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5029695162, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4585033402, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 983283962, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4173747578, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175553914, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175471994, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3715812410, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1923202106, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 750170362, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175560122, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4175564474, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4174685562, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030590650, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.11.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3496366842, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3480341178, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.11.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1704027194, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2836733946, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030557370, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4164457530, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5032233658, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1935785018, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5030922938, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2145795130, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 477141050, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 469995578, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 476289082, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 494704698, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.12.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3471946298, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.12.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 455962298, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 27619002, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 254111418, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 455438010, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 749381882, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 5031447226, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3496403706, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 9924282, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 756461818, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1237252986, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 469929914, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2147105850, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4173813114, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 456249274, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 457347002, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1706456122, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 456244922, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 469997626, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.13.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 8979898, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.13.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 8975802, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2675267798, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 3944952890, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 8451514, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3609442902, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1782141654, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3137646934, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3115725078, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2452850774, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 9246138, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 9825850, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 9246202, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 990640442, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 9252410, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1657209850, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 1440416762, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 4165243962, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00002-of-00006.safetensors", |
| "offset_bytes": 2349301626, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.14.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5031569686, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.14.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3397575190, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2914398486, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3397579286, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3598905878, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3607851542, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 894496598, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3116544278, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3610262102, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 955792022, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3189627350, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2654177430, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 896888726, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3181089238, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2422914582, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2440005142, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2011525846, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3864642198, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3189627414, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2663712982, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 906197078, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.15.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1791320790, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1992651478, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.15.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1782928086, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 907541078, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2018112214, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2219480278, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1580810966, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1165556374, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 893708118, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 905410646, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 920123990, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 896921622, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 938228374, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2018149590, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.16.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1782137558, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.16.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2219476182, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5032687958, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3617340054, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 892921686, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3853828758, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3844618902, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3864675222, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3855467158, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2220270806, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3608637974, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5032622422, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5250333526, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3116546326, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4524473750, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4753243798, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1373176470, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2422910230, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1366885014, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.17.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4087977302, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.17.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4087981398, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4524588438, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4088001878, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2654242966, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3843832470, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4743249558, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4766088854, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4744035990, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4314758550, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2654177366, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3864677270, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4081669014, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4081619862, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3871820694, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2893424918, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3865529238, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4081669078, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4081685718, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.18.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3116595478, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.18.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2220266710, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4821000406, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1791324886, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3180302806, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4779300822, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4820066518, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5031043350, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2422123798, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3880258454, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4087977174, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2675202262, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 937966166, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4779249622, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4766353366, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3116601686, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3611048534, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4780089302, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4780122326, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5031835990, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5250333014, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.19.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5250333590, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5244041558, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.19.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2422910486, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4766402518, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2452850262, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 5234833750, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1379484310, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 691578710, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2663155862, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 471869206, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 472688406, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2674381014, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 906721878, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 481863446, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.20.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 457705238, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.20.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2018145494, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 29345558, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3196213782, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 457180950, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 27772694, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 28559126, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3171355094, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 26953494, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 247449366, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2663712918, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 471738134, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4289328470, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 464264982, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 464316182, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 1998942934, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 464922390, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4726472086, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 457971478, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.21.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3129252182, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.21.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 26949398, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 978479608, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1192391224, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 2672920790, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1911221368, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1912091768, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1394278968, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 761422264, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1655790712, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 3137714582, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4820885718, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 691578646, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 6633110, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 6682262, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 14333590, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 6993558, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 4766090902, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00003-of-00006.safetensors", |
| "offset_bytes": 341654, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.22.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 473769336, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.22.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2129454200, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2380574648, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2143068024, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2142543736, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3230121976, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1910434936, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1857150072, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 760633784, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2581901240, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 482760120, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3231694840, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3230089144, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2131065976, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 978430456, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1634753592, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1621875256, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2140446328, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1921266808, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2361436024, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1634753080, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.23.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1927562360, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 972057080, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.23.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1927558264, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3464708216, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2131461240, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2808410040, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 762210744, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1403771448, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2129493112, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1909632120, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3944981048, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 762208696, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2141757304, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 500422200, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.24.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3954680376, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.24.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 473765240, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3027875768, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2809196472, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3010523064, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3230908408, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3229237176, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4815406200, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2130017400, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1927566456, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 708302392, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3910818296, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 482495864, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1912040568, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 474035576, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3442294840, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1903307896, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 751687864, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 702010936, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.25.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3492104440, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.25.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1394274872, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 5018422456, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3709491704, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2131494520, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2379788216, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3492108536, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3927661048, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1394315832, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3507837432, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1179806200, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4588782648, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3492104376, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3230039992, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3483615352, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1179806264, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3477291128, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3507575032, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1628166712, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.26.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 482489720, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.26.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2140440184, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4359128120, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4614014072, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4586931256, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4567270456, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4587455544, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 482493816, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3433119800, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4815981752, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4157801144, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4157735608, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4588848184, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3956359864, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 978381304, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4574348344, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3501283576, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4359127864, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4560454712, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 5017308344, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 742446712, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.27.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 5219765944, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4568056888, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.27.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 5219770040, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 1865540728, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 5219749048, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2140970872, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 62166392, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 271881592, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 473240952, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 43746680, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 44602744, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 43679096, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3454910520, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3231793208, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.28.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 44565880, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.28.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 17343416, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2547471498, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 4157801208, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 16819128, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2750145738, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1901136970, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2547436682, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1832006282, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3956409016, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 24163512, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 43613432, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 3483666552, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 2131117176, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 24165624, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 31030520, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 24214776, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 24163256, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00004-of-00006.safetensors", |
| "offset_bytes": 17871800, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.29.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3370276554, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.29.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1892432970, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1621698826, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2750932170, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2749621450, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3395512010, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 920470730, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4221110090, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2546617482, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3168949962, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1883978762, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1834439562, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1902496842, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1409968330, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1877636106, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1865053194, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1910918282, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1832792714, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1877687306, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.30.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952920778, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.30.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1408915658, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1189304266, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1917209738, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1832792970, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 969576394, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3598477002, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2749619402, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1410869450, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1411918026, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4011390730, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1824600394, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2749324426, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1390663626, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1844016138, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1844081674, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 956927946, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1400526794, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3154269898, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 942999306, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 942802122, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.31.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1844009930, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 963219402, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.31.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1834367882, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 943785738, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952920266, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1188515786, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4011390794, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2118536330, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 929694922, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4230883146, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4229764938, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4231702346, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3808978634, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3810060042, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.32.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4230879050, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.32.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4665655178, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952941258, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4675100554, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4666183562, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4956457482, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4935409610, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952258762, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1823289674, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4440946570, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1621698762, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1621633226, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4936441866, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4659312522, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4434081674, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 930219210, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4434130826, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 32, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5250300490, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.32.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 32, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4659363722, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.33.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4666707850, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.33.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952326346, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5008951818, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2319862922, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4982671882, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5227055626, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1399740362, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5228399114, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1824928330, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4232230794, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4956457162, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1834371978, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4956457098, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4956403850, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 2952260810, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4936441994, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4949549194, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 33, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4956457226, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.33.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 33, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4963535370, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.34.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5249720906, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.34.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4221105994, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 492586186, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3397117642, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4433557386, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 465814730, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 466633930, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 464960714, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 464995530, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 702301386, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5250337674, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5228401226, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5250335562, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4936196042, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5249725002, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 5228468810, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4957243914, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 34, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1824665930, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.34.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 34, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1825714826, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1408919754, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 35, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444676170, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.35.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 35, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444672074, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444939338, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.35.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 35, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4011386634, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 452377674, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 35, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444676682, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.35.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1410017482, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 3599263434, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 978800586, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444147786, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 224405322, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 225224522, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 223551306, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 223618890, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 35, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 242788170, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.36.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 209798858, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.36.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 209794762, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1783720371, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 8435402, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4440422282, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2208721843, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 494200819, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2451508339, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2217978803, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1993960371, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 216618698, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 223485770, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 216618762, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 1391237066, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 216620874, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 4970088970, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 217194314, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 36, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 444114762, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.36.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 36, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00005-of-00006.safetensors", |
| "offset_bytes": 210327242, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.37.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 504201203, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.37.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 503408627, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 67233779, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 276981747, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1985046963, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 503412723, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 504205299, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 504199155, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2207935411, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 746267059, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2665522291, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1155228083, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 512104691, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 729325939, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2217929651, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2195352499, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1148936627, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 37, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2673124659, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.37.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 37, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 505549043, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.38.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2673126963, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.38.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2673131059, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 512104755, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2242542707, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1155621299, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1765370291, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2899627571, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2673124915, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1782933939, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2689912371, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2672600307, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2220981363, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2665522355, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2451510387, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 32, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2220899315, |
| "length_bytes": 49152 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2221046899, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 4096, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2444426355, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 38, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 505253875, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.38.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 38, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2665522419, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1156145587, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 39, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1985571251, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.39.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 39, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2219846643, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 713433459, |
| "length_bytes": 6291456 |
| }, |
| { |
| "name": "language_model.model.layers.39.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 39, |
| "dtype": "bf16", |
| "shape": [ |
| 2048 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2901301427, |
| "length_bytes": 4096 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2652918899, |
| "length_bytes": 12582912 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 39, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2901006003, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.39.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 720019827, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight", |
| "role": "ffn_down_exps", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2451559539, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight", |
| "role": "ffn_gate_exps", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1538351539, |
| "length_bytes": 201326592 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.gate.weight", |
| "role": "ffn_gate_inp", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 256, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 1748068787, |
| "length_bytes": 524288 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.shared_expert.down_proj.weight", |
| "role": "ffn_shared_expert_down", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 2048, |
| 96 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2219850739, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.shared_expert.gate_proj.weight", |
| "role": "ffn_shared_expert_gate", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 2671813875, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.shared_expert_gate.weight", |
| "role": "ffn_shared_expert_gate_inp", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 1, |
| 512 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 15917043, |
| "length_bytes": 2048 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.shared_expert.up_proj.weight", |
| "role": "ffn_shared_expert_up", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 15951859, |
| "length_bytes": 786432 |
| }, |
| { |
| "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight", |
| "role": "ffn_up_exps", |
| "layer_index": 39, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 6 |
| }, |
| "shape": [ |
| 256, |
| 512, |
| 384 |
| ], |
| "file": "model-00006-of-00006.safetensors", |
| "offset_bytes": 947610035, |
| "length_bytes": 201326592 |
| } |
| ] |
| } |
