{ "schema_version": "ax.native_model.v1", "model_family": "unlimited_ocr", "tensor_format": "safetensors", "layer_count": 12, "hidden_size": 1280, "intermediate_size": 6848, "attention_head_count": 10, "attention_head_dim": 128, "kv_head_count": 10, "vocab_size": 129280, "tie_word_embeddings": false, "no_rope_layer_interval": 0, "intermediate_size_mlp": 0, "attn_output_gate": false, "rms_norm_eps": 0.000001, "moe_norm_topk_prob": false, "hidden_size_per_layer_input": 0, "moe": { "expert_count": 64, "experts_per_token": 6, "expert_intermediate_size": 896, "layer_freq": 1, "first_dense_layers": 1, "shared_expert_count": 2, "sigmoid_routing": false }, "tensors": [ { "name": "language_model.model.norm.weight", "role": "final_norm", "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 175916953, "length_bytes": 2560 }, { "name": "language_model.lm_head.weight", "role": "lm_head", "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 8 }, "shape": [ 129280, 320 ], "file": "model.safetensors", "offset_bytes": 10438553, "length_bytes": 165478400 }, { "name": "language_model.model.embed_tokens.weight", "role": "token_embedding", "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 129280, 320 ], "file": "model.safetensors", "offset_bytes": 1756151193, "length_bytes": 165478400 }, { "name": "image_newline", "role": "unlimited_ocr_image_newline", "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 93593, "length_bytes": 2560 }, { "name": "projector.layers.weight", "role": "unlimited_ocr_projector", "dtype": "bf16", "shape": [ 1280, 2048 ], "file": "model.safetensors", "offset_bytes": 2562722713, "length_bytes": 5242880 }, { "name": "view_separator", "role": "unlimited_ocr_view_separator", "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 3039130137, "length_bytes": 2560 }, { "name": "language_model.model.layers.0.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2350949657, "length_bytes": 1638400 }, { "name": "language_model.model.layers.0.input_layernorm.weight", "role": "attention_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1718457753, "length_bytes": 2560 }, { "name": "language_model.model.layers.0.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3606229785, "length_bytes": 1638400 }, { "name": "language_model.model.layers.0.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1718455193, "length_bytes": 2560 }, { "name": "language_model.model.layers.0.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3342692889, "length_bytes": 1638400 }, { "name": "language_model.model.layers.0.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3027135001, "length_bytes": 1638400 }, { "name": "language_model.model.layers.0.mlp.down_proj.weight", "role": "ffn_down", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 1712 ], "file": "model.safetensors", "offset_bytes": 1720372633, "length_bytes": 8765440 }, { "name": "language_model.model.layers.0.mlp.gate_proj.weight", "role": "ffn_gate", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 6848, 320 ], "file": "model.safetensors", "offset_bytes": 1742109593, "length_bytes": 8765440 }, { "name": "language_model.model.layers.0.mlp.up_proj.weight", "role": "ffn_up", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 6848, 320 ], "file": "model.safetensors", "offset_bytes": 1733344153, "length_bytes": 8765440 }, { "name": "language_model.model.layers.1.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2181890585, "length_bytes": 1638400 }, { "name": "language_model.model.layers.1.input_layernorm.weight", "role": "attention_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1485049753, "length_bytes": 2560 }, { "name": "language_model.model.layers.1.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1718460313, "length_bytes": 1638400 }, { "name": "language_model.model.layers.1.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 3319586969, "length_bytes": 2560 }, { "name": "language_model.model.layers.1.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1716816793, "length_bytes": 1638400 }, { "name": "language_model.model.layers.1.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2808163225, "length_bytes": 1638400 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 1565059993, "length_bytes": 73400320 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 3105961241, "length_bytes": 73400320 }, { "name": "language_model.model.layers.1.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 1, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 1489202073, "length_bytes": 163840 }, { "name": "language_model.model.layers.1.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 3561485593, "length_bytes": 2293760 }, { "name": "language_model.model.layers.1.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1485269913, "length_bytes": 2293760 }, { "name": "language_model.model.layers.1.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 3815614873, "length_bytes": 2293760 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 1638460313, "length_bytes": 73400320 }, { "name": "language_model.model.layers.2.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3352739865, "length_bytes": 1638400 }, { "name": "language_model.model.layers.2.input_layernorm.weight", "role": "attention_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1317443993, "length_bytes": 2560 }, { "name": "language_model.model.layers.2.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1478391193, "length_bytes": 1638400 }, { "name": "language_model.model.layers.2.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 2723929625, "length_bytes": 2560 }, { "name": "language_model.model.layers.2.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1483408793, "length_bytes": 1638400 }, { "name": "language_model.model.layers.2.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1481719193, "length_bytes": 1638400 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 2943153689, "length_bytes": 73400320 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 2369328921, "length_bytes": 73400320 }, { "name": "language_model.model.layers.2.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 2, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 1478227353, "length_bytes": 163840 }, { "name": "language_model.model.layers.2.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 1317520793, "length_bytes": 2293760 }, { "name": "language_model.model.layers.2.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1395580313, "length_bytes": 2293760 }, { "name": "language_model.model.layers.2.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1393286553, "length_bytes": 2293760 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 1402461593, "length_bytes": 73400320 }, { "name": "language_model.model.layers.3.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2607808665, "length_bytes": 1638400 }, { "name": "language_model.model.layers.3.input_layernorm.weight", "role": "attention_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1159653273, "length_bytes": 2560 }, { "name": "language_model.model.layers.3.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1480029593, "length_bytes": 1638400 }, { "name": "language_model.model.layers.3.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 2633741849, "length_bytes": 2560 }, { "name": "language_model.model.layers.3.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1315805593, "length_bytes": 1638400 }, { "name": "language_model.model.layers.3.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2832770841, "length_bytes": 1638400 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 2258578329, "length_bytes": 73400320 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 3225070361, "length_bytes": 73400320 }, { "name": "language_model.model.layers.3.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 3, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 1716601753, "length_bytes": 163840 }, { "name": "language_model.model.layers.3.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 1159727513, "length_bytes": 2293760 }, { "name": "language_model.model.layers.3.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1164315033, "length_bytes": 2293760 }, { "name": "language_model.model.layers.3.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1162021273, "length_bytes": 2293760 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 1166608793, "length_bytes": 73400320 }, { "name": "language_model.model.layers.4.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1156376473, "length_bytes": 1638400 }, { "name": "language_model.model.layers.4.input_layernorm.weight", "role": "attention_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1317446553, "length_bytes": 2560 }, { "name": "language_model.model.layers.4.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1154686873, "length_bytes": 1638400 }, { "name": "language_model.model.layers.4.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 2183535513, "length_bytes": 2560 }, { "name": "language_model.model.layers.4.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1158014873, "length_bytes": 1638400 }, { "name": "language_model.model.layers.4.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2935148569, "length_bytes": 1638400 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 1930365721, "length_bytes": 73400320 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 2461558297, "length_bytes": 73400320 }, { "name": "language_model.model.layers.4.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 4, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 1076381593, "length_bytes": 163840 }, { "name": "language_model.model.layers.4.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 1069500313, "length_bytes": 2293760 }, { "name": "language_model.model.layers.4.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1074087833, "length_bytes": 2293760 }, { "name": "language_model.model.layers.4.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1071794073, "length_bytes": 2293760 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 1078890393, "length_bytes": 73400320 }, { "name": "language_model.model.layers.5.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1067790233, "length_bytes": 1638400 }, { "name": "language_model.model.layers.5.input_layernorm.weight", "role": "attention_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 830506393, "length_bytes": 2560 }, { "name": "language_model.model.layers.5.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 991061913, "length_bytes": 1638400 }, { "name": "language_model.model.layers.5.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 830503833, "length_bytes": 2560 }, { "name": "language_model.model.layers.5.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2014376217, "length_bytes": 1638400 }, { "name": "language_model.model.layers.5.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1066151833, "length_bytes": 1638400 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 837984153, "length_bytes": 73400320 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 915971993, "length_bytes": 73400320 }, { "name": "language_model.model.layers.5.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 5, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 835362713, "length_bytes": 163840 }, { "name": "language_model.model.layers.5.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 2830401945, "length_bytes": 2293760 }, { "name": "language_model.model.layers.5.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 833017753, "length_bytes": 2293760 }, { "name": "language_model.model.layers.5.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 830652313, "length_bytes": 2293760 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 2185178009, "length_bytes": 73400320 }, { "name": "language_model.model.layers.6.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3507597081, "length_bytes": 1638400 }, { "name": "language_model.model.layers.6.input_layernorm.weight", "role": "attention_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 749738393, "length_bytes": 2560 }, { "name": "language_model.model.layers.6.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 825486233, "length_bytes": 1638400 }, { "name": "language_model.model.layers.6.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1485195673, "length_bytes": 2560 }, { "name": "language_model.model.layers.6.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 828865433, "length_bytes": 1638400 }, { "name": "language_model.model.layers.6.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 827227033, "length_bytes": 1638400 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 752034713, "length_bytes": 73400320 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 2642134553, "length_bytes": 73400320 }, { "name": "language_model.model.layers.6.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 6, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 835526553, "length_bytes": 163840 }, { "name": "language_model.model.layers.6.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 749740953, "length_bytes": 2293760 }, { "name": "language_model.model.layers.6.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 1714154393, "length_bytes": 2293760 }, { "name": "language_model.model.layers.6.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 3198452505, "length_bytes": 2293760 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 3434024729, "length_bytes": 73400320 }, { "name": "language_model.model.layers.7.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 748048793, "length_bytes": 1638400 }, { "name": "language_model.model.layers.7.input_layernorm.weight", "role": "attention_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 3392695705, "length_bytes": 2560 }, { "name": "language_model.model.layers.7.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2818202521, "length_bytes": 1638400 }, { "name": "language_model.model.layers.7.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 743284633, "length_bytes": 2560 }, { "name": "language_model.model.layers.7.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1487563673, "length_bytes": 1638400 }, { "name": "language_model.model.layers.7.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2183539609, "length_bytes": 1638400 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 1319886233, "length_bytes": 73400320 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 2861748249, "length_bytes": 73400320 }, { "name": "language_model.model.layers.7.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 7, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 3507431193, "length_bytes": 163840 }, { "name": "language_model.model.layers.7.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 3179375897, "length_bytes": 2293760 }, { "name": "language_model.model.layers.7.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 3417343897, "length_bytes": 2293760 }, { "name": "language_model.model.layers.7.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 743358873, "length_bytes": 2293760 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 3642288409, "length_bytes": 73400320 }, { "name": "language_model.model.layers.8.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 989423513, "length_bytes": 1638400 }, { "name": "language_model.model.layers.8.input_layernorm.weight", "role": "attention_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 3342675993, "length_bytes": 2560 }, { "name": "language_model.model.layers.8.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2364151577, "length_bytes": 1638400 }, { "name": "language_model.model.layers.8.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 578612633, "length_bytes": 2560 }, { "name": "language_model.model.layers.8.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3632181529, "length_bytes": 1638400 }, { "name": "language_model.model.layers.8.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 1729138073, "length_bytes": 1638400 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 992751513, "length_bytes": 73400320 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 668194713, "length_bytes": 73400320 }, { "name": "language_model.model.layers.8.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 8, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 2571522841, "length_bytes": 163840 }, { "name": "language_model.model.layers.8.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 578738073, "length_bytes": 2293760 }, { "name": "language_model.model.layers.8.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 583325593, "length_bytes": 2293760 }, { "name": "language_model.model.layers.8.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 581031833, "length_bytes": 2293760 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 592500633, "length_bytes": 73400320 }, { "name": "language_model.model.layers.9.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 576923033, "length_bytes": 1638400 }, { "name": "language_model.model.layers.9.input_layernorm.weight", "role": "attention_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1485047193, "length_bytes": 2560 }, { "name": "language_model.model.layers.9.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 741646233, "length_bytes": 1638400 }, { "name": "language_model.model.layers.9.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 3405807001, "length_bytes": 2560 }, { "name": "language_model.model.layers.9.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3559845145, "length_bytes": 1638400 }, { "name": "language_model.model.layers.9.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 575233433, "length_bytes": 1638400 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 2102578713, "length_bytes": 73400320 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 501730713, "length_bytes": 73400320 }, { "name": "language_model.model.layers.9.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 9, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 419646873, "length_bytes": 163840 }, { "name": "language_model.model.layers.9.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 2012082457, "length_bytes": 2293760 }, { "name": "language_model.model.layers.9.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 3760249753, "length_bytes": 2293760 }, { "name": "language_model.model.layers.9.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 590206873, "length_bytes": 2293760 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 426036633, "length_bytes": 73400320 }, { "name": "language_model.model.layers.10.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3813974425, "length_bytes": 1638400 }, { "name": "language_model.model.layers.10.input_layernorm.weight", "role": "attention_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 1750875033, "length_bytes": 2560 }, { "name": "language_model.model.layers.10.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 416103833, "length_bytes": 1638400 }, { "name": "language_model.model.layers.10.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 330873753, "length_bytes": 2560 }, { "name": "language_model.model.layers.10.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 417793433, "length_bytes": 1638400 }, { "name": "language_model.model.layers.10.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 2543350297, "length_bytes": 1638400 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 1242405273, "length_bytes": 73400320 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 1491659673, "length_bytes": 73400320 }, { "name": "language_model.model.layers.10.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 10, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 335607193, "length_bytes": 163840 }, { "name": "language_model.model.layers.10.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 3796838297, "length_bytes": 2293760 }, { "name": "language_model.model.layers.10.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 333313433, "length_bytes": 2293760 }, { "name": "language_model.model.layers.10.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 330947993, "length_bytes": 2293760 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 340358553, "length_bytes": 73400320 }, { "name": "language_model.model.layers.11.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 329235353, "length_bytes": 1638400 }, { "name": "language_model.model.layers.11.input_layernorm.weight", "role": "attention_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 2177161753, "length_bytes": 2560 }, { "name": "language_model.model.layers.11.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 3362843673, "length_bytes": 1638400 }, { "name": "language_model.model.layers.11.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 1280 ], "file": "model.safetensors", "offset_bytes": 175919513, "length_bytes": 2560 }, { "name": "language_model.model.layers.11.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 424398233, "length_bytes": 1638400 }, { "name": "language_model.model.layers.11.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 320 ], "file": "model.safetensors", "offset_bytes": 254196633, "length_bytes": 1638400 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 1280, 224 ], "file": "model.safetensors", "offset_bytes": 2020812953, "length_bytes": 73400320 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 180745113, "length_bytes": 73400320 }, { "name": "language_model.model.layers.11.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 11, "dtype": "bf16", "shape": [ 64, 1280 ], "file": "model.safetensors", "offset_bytes": 178287513, "length_bytes": 163840 }, { "name": "language_model.model.layers.11.mlp.shared_experts.down_proj.weight", "role": "ffn_shared_expert_down", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1280, 448 ], "file": "model.safetensors", "offset_bytes": 3515535129, "length_bytes": 2293760 }, { "name": "language_model.model.layers.11.mlp.shared_experts.gate_proj.weight", "role": "ffn_shared_expert_gate", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 3720423705, "length_bytes": 2293760 }, { "name": "language_model.model.layers.11.mlp.shared_experts.up_proj.weight", "role": "ffn_shared_expert_up", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 1792, 320 ], "file": "model.safetensors", "offset_bytes": 175993753, "length_bytes": 2293760 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "mxfp8", "group_size": 32, "bits": 8 }, "shape": [ 64, 896, 320 ], "file": "model.safetensors", "offset_bytes": 255835033, "length_bytes": 73400320 } ] }