{ "schema_version": "ax.native_model.v1", "model_family": "qwen3_vl_moe", "tensor_format": "safetensors", "layer_count": 48, "hidden_size": 2048, "intermediate_size": 6144, "attention_head_count": 32, "attention_head_dim": 128, "kv_head_count": 4, "vocab_size": 151936, "tie_word_embeddings": false, "rope_theta": 5000000, "rope_scaling_type": "default", "no_rope_layer_interval": 0, "intermediate_size_mlp": 0, "attn_output_gate": false, "rms_norm_eps": 0.000001, "moe_norm_topk_prob": true, "hidden_size_per_layer_input": 0, "moe": { "expert_count": 128, "experts_per_token": 8, "expert_intermediate_size": 768, "sigmoid_routing": false }, "think_start_token_id": 151667, "think_end_token_id": 151668, "dropped_tensors": { "count": 770, "media_role_hits": 0, "names_sample": [ "language_model.model.embed_tokens.biases", "language_model.model.embed_tokens.scales", "language_model.model.layers.0.mlp.gate.biases", "language_model.model.layers.0.mlp.gate.scales", "language_model.model.layers.0.mlp.switch_mlp.down_proj.biases", "language_model.model.layers.0.mlp.switch_mlp.down_proj.scales", "language_model.model.layers.0.mlp.switch_mlp.gate_proj.biases", "language_model.model.layers.0.mlp.switch_mlp.gate_proj.scales", "language_model.model.layers.0.mlp.switch_mlp.up_proj.biases", "language_model.model.layers.0.mlp.switch_mlp.up_proj.scales", "language_model.model.layers.0.self_attn.k_proj.biases", "language_model.model.layers.0.self_attn.k_proj.scales", "language_model.model.layers.0.self_attn.o_proj.biases", "language_model.model.layers.0.self_attn.o_proj.scales", "language_model.model.layers.0.self_attn.q_proj.biases", "language_model.model.layers.0.self_attn.q_proj.scales" ] }, "tensors": [ { "name": "language_model.model.norm.weight", "role": "final_norm", "dtype": "bf16", "shape": [ 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 945943096, "length_bytes": 4096 }, { "name": "language_model.lm_head.weight", "role": "lm_head", "dtype": "bf16", "shape": [ 151936, 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 8504, "length_bytes": 622329856 }, { "name": "vision_tower.blocks.0.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5322117118, "length_bytes": 2304 }, { "name": "vision_tower.blocks.0.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5239497726, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.0.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5199203262, "length_bytes": 6912 }, { "name": "vision_tower.blocks.0.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3869606014, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.0.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4939956222, "length_bytes": 8608 }, { "name": "vision_tower.blocks.0.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5345147902, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.0.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4443567934, "length_bytes": 2304 }, { "name": "vision_tower.blocks.0.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3785527006, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.0.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5345145598, "length_bytes": 2304 }, { "name": "vision_tower.blocks.0.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5202653118, "length_bytes": 2304 }, { "name": "vision_tower.blocks.0.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5292102398, "length_bytes": 2304 }, { "name": "vision_tower.blocks.0.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5306353406, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252868094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5095878846, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.1.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698569502, "length_bytes": 6912 }, { "name": "vision_tower.blocks.1.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5336644862, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.1.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4888883134, "length_bytes": 8608 }, { "name": "vision_tower.blocks.1.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4869948606, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.1.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252079358, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4294177790, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.1.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5221528414, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5253656830, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4907572222, "length_bytes": 2304 }, { "name": "vision_tower.blocks.1.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5367651838, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4672244094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5319462910, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.10.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4314021534, "length_bytes": 6912 }, { "name": "vision_tower.blocks.10.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4979996894, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.10.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511550430, "length_bytes": 8608 }, { "name": "vision_tower.blocks.10.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4734967838, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.10.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4970050462, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4634520702, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.10.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4888900350, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4672779902, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4672253310, "length_bytes": 2304 }, { "name": "vision_tower.blocks.10.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460163550, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847082718, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4508880606, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.11.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5292104702, "length_bytes": 6912 }, { "name": "vision_tower.blocks.11.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511561342, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.11.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713219902, "length_bytes": 8608 }, { "name": "vision_tower.blocks.11.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4498964190, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.11.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4498961886, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3393456030, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.11.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4711236894, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5344871934, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4950407838, "length_bytes": 2304 }, { "name": "vision_tower.blocks.11.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4620262014, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4479998686, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4480012510, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.12.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4482666974, "length_bytes": 6912 }, { "name": "vision_tower.blocks.12.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4880920510, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.12.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229464862, "length_bytes": 8608 }, { "name": "vision_tower.blocks.12.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4470082270, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.12.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5367647230, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460165854, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.12.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4880389310, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698584606, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4495256798, "length_bytes": 2304 }, { "name": "vision_tower.blocks.12.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3869603710, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4443570238, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4456182494, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.13.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4459360990, "length_bytes": 6912 }, { "name": "vision_tower.blocks.13.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5077473214, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.13.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4424617118, "length_bytes": 8608 }, { "name": "vision_tower.blocks.13.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4970052766, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.13.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4424614814, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4701306654, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.13.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4673374334, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460161246, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4664279166, "length_bytes": 2304 }, { "name": "vision_tower.blocks.13.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3506707358, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4880918206, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5074819006, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.14.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4284254462, "length_bytes": 6912 }, { "name": "vision_tower.blocks.14.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3556172606, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.14.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4274329438, "length_bytes": 8608 }, { "name": "vision_tower.blocks.14.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5266250238, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.14.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4157718462, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3765685982, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.14.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4424612510, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5345136382, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5231521886, "length_bytes": 2304 }, { "name": "vision_tower.blocks.14.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4304094206, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4664276862, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4142474942, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.15.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4672246398, "length_bytes": 6912 }, { "name": "vision_tower.blocks.15.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5231535102, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.15.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4979969182, "length_bytes": 8608 }, { "name": "vision_tower.blocks.15.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4132558270, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.15.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4907574526, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4100129310, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.15.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5074816702, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511534814, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4157712062, "length_bytes": 2304 }, { "name": "vision_tower.blocks.15.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5028104382, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3941622046, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3959515678, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.16.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3962701086, "length_bytes": 6912 }, { "name": "vision_tower.blocks.16.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4063371294, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.16.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4456164670, "length_bytes": 8608 }, { "name": "vision_tower.blocks.16.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4644444030, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.16.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3929032222, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4304105118, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.16.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2567866398, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4087544094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190925342, "length_bytes": 2304 }, { "name": "vision_tower.blocks.16.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4077625374, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925298462, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3200859678, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.17.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4987967710, "length_bytes": 6912 }, { "name": "vision_tower.blocks.17.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4626558078, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.17.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925281662, "length_bytes": 8608 }, { "name": "vision_tower.blocks.17.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5202655422, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.17.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460158942, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3912711038, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.17.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3928238878, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3928241182, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925307166, "length_bytes": 2304 }, { "name": "vision_tower.blocks.17.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3928236574, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4266364510, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3909986686, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.18.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4711223070, "length_bytes": 6912 }, { "name": "vision_tower.blocks.18.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4907579134, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.18.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5004184798, "length_bytes": 8608 }, { "name": "vision_tower.blocks.18.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3350407678, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.18.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3869601406, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3877568638, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.18.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3912640894, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3912708734, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5231532798, "length_bytes": 2304 }, { "name": "vision_tower.blocks.18.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698560286, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3928243486, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3866942590, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.19.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4620255102, "length_bytes": 6912 }, { "name": "vision_tower.blocks.19.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4664281470, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.19.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847098846, "length_bytes": 8608 }, { "name": "vision_tower.blocks.19.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847109758, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.19.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847087326, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3900070270, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.19.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925300766, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3869599102, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460156638, "length_bytes": 2304 }, { "name": "vision_tower.blocks.19.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3869596798, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5284129278, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5212582750, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.2.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5221530718, "length_bytes": 6912 }, { "name": "vision_tower.blocks.2.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5276166654, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.2.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5005515646, "length_bytes": 8608 }, { "name": "vision_tower.blocks.2.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4284261374, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.2.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5202650814, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5242151934, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.2.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5231454046, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3393451166, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3507238558, "length_bytes": 2304 }, { "name": "vision_tower.blocks.2.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5212571838, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3795443422, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3812177886, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.20.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3962169886, "length_bytes": 6912 }, { "name": "vision_tower.blocks.20.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4258399582, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.20.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5028095774, "length_bytes": 8608 }, { "name": "vision_tower.blocks.20.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3775602398, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.20.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5330088958, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713228510, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.20.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3135870494, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3828206046, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3814832094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.20.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5005511038, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4443565630, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3601858110, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.21.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3705175614, "length_bytes": 6912 }, { "name": "vision_tower.blocks.21.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3705182526, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.21.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3360324094, "length_bytes": 8608 }, { "name": "vision_tower.blocks.21.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3591941694, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.21.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847107454, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3564139838, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.21.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713212990, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229747134, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4719726110, "length_bytes": 2304 }, { "name": "vision_tower.blocks.21.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713210686, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5367649534, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3922627454, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.22.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4456157758, "length_bytes": 6912 }, { "name": "vision_tower.blocks.22.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3548144446, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.22.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3519813790, "length_bytes": 8608 }, { "name": "vision_tower.blocks.22.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3519822398, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.22.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3507240862, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3509897374, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.22.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3564137534, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4744952094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3564135230, "length_bytes": 2304 }, { "name": "vision_tower.blocks.22.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2542161182, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3393453470, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925320222, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.23.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3403372446, "length_bytes": 6912 }, { "name": "vision_tower.blocks.23.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5322126334, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.23.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4110045726, "length_bytes": 8608 }, { "name": "vision_tower.blocks.23.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3802261470, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.23.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4753068222, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3383534750, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.23.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229473470, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3507236254, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3403379358, "length_bytes": 2304 }, { "name": "vision_tower.blocks.23.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3506709662, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4443572542, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3360332702, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.24.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4644437118, "length_bytes": 6912 }, { "name": "vision_tower.blocks.24.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3375569822, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.24.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229754046, "length_bytes": 8608 }, { "name": "vision_tower.blocks.24.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3340491262, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.24.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713215294, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4274338046, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.24.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3591935294, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3941619742, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3383532446, "length_bytes": 2304 }, { "name": "vision_tower.blocks.24.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2542163486, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3239689982, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3403381662, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.25.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5098533054, "length_bytes": 6912 }, { "name": "vision_tower.blocks.25.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3574056254, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.25.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3239681374, "length_bytes": 8608 }, { "name": "vision_tower.blocks.25.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4860029886, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.25.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229751742, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229764958, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.25.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3239760126, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229749438, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3239692286, "length_bytes": 2304 }, { "name": "vision_tower.blocks.25.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925317662, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3537701438, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3213430302, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.26.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5345138686, "length_bytes": 6912 }, { "name": "vision_tower.blocks.26.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5284139774, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.26.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5212574142, "length_bytes": 8608 }, { "name": "vision_tower.blocks.26.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3203513886, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.26.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190940958, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190943262, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.26.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3228673822, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229462558, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5095876542, "length_bytes": 2304 }, { "name": "vision_tower.blocks.26.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3228671518, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3900067966, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5199210174, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.3.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698562590, "length_bytes": 6912 }, { "name": "vision_tower.blocks.3.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4266366814, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.3.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4979977790, "length_bytes": 8608 }, { "name": "vision_tower.blocks.3.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4110054334, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.3.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2426938142, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5221537630, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.3.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252077054, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5040691902, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4869946302, "length_bytes": 2304 }, { "name": "vision_tower.blocks.3.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4620252798, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5040689598, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5040698302, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.4.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5322119422, "length_bytes": 6912 }, { "name": "vision_tower.blocks.4.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3529738814, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.4.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4258390974, "length_bytes": 8608 }, { "name": "vision_tower.blocks.4.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4940491422, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.4.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847085022, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5018172958, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.4.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4620264318, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5055937726, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2567864094, "length_bytes": 2304 }, { "name": "vision_tower.blocks.4.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2993480478, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4997891038, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4719728414, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.5.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511541214, "length_bytes": 6912 }, { "name": "vision_tower.blocks.5.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4744954398, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.5.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4888891742, "length_bytes": 8608 }, { "name": "vision_tower.blocks.5.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4987974622, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.5.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3713217598, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4654360446, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.5.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4979986398, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5005513342, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511559038, "length_bytes": 2304 }, { "name": "vision_tower.blocks.5.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252072446, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5055940030, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4895194110, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.6.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4258384062, "length_bytes": 6912 }, { "name": "vision_tower.blocks.6.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4915541758, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.6.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4304096510, "length_bytes": 8608 }, { "name": "vision_tower.blocks.6.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3582018878, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.6.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4119973054, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4960134046, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.6.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4939953918, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4939964830, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252074750, "length_bytes": 2304 }, { "name": "vision_tower.blocks.6.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3929034526, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4119970750, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3507243166, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.7.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5005504126, "length_bytes": 6912 }, { "name": "vision_tower.blocks.7.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4931991294, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.7.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5231524190, "length_bytes": 8608 }, { "name": "vision_tower.blocks.7.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4077627678, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.7.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4880391614, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5295650558, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.7.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5344607486, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4907576830, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3097843998, "length_bytes": 2304 }, { "name": "vision_tower.blocks.7.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4460154334, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3828203742, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4495783390, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.8.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4752991166, "length_bytes": 6912 }, { "name": "vision_tower.blocks.8.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4923504382, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.8.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4752982558, "length_bytes": 8608 }, { "name": "vision_tower.blocks.8.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5085960126, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.8.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4744949790, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3857026174, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.8.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4753070526, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4860027582, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4752998078, "length_bytes": 2304 }, { "name": "vision_tower.blocks.8.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4753065918, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.attn.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4480010206, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.attn.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698652446, "length_bytes": 2654208 }, { "name": "vision_tower.blocks.9.attn.qkv.bias", "role": "other", "dtype": "bf16", "shape": [ 3456 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4711229982, "length_bytes": 6912 }, { "name": "vision_tower.blocks.9.attn.qkv.weight", "role": "other", "dtype": "bf16", "shape": [ 3456, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4711239198, "length_bytes": 7962624 }, { "name": "vision_tower.blocks.9.mlp.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4685959550, "length_bytes": 8608 }, { "name": "vision_tower.blocks.9.mlp.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3537703742, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.9.mlp.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4673306494, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.mlp.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 4304 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4414691998, "length_bytes": 9916416 }, { "name": "vision_tower.blocks.9.norm1.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511548126, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.norm1.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5055935422, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.norm2.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5028093470, "length_bytes": 2304 }, { "name": "vision_tower.blocks.9.norm2.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4734965534, "length_bytes": 2304 }, { "name": "vision_tower.deepstack_merger_list.0.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4480000990, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.0.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4608, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3723218654, "length_bytes": 42467328 }, { "name": "vision_tower.deepstack_merger_list.0.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3097846302, "length_bytes": 4096 }, { "name": "vision_tower.deepstack_merger_list.0.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 2048, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5055942334, "length_bytes": 18874368 }, { "name": "vision_tower.deepstack_merger_list.0.norm.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3110433310, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.0.norm.weight", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229475774, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.1.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3847089630, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.1.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4608, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3042784542, "length_bytes": 42467328 }, { "name": "vision_tower.deepstack_merger_list.1.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925303070, "length_bytes": 4096 }, { "name": "vision_tower.deepstack_merger_list.1.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 2048, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4424691262, "length_bytes": 18874368 }, { "name": "vision_tower.deepstack_merger_list.1.norm.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4698551070, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.1.norm.weight", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3091543326, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.2.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190927646, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.2.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4608, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2999774238, "length_bytes": 42467328 }, { "name": "vision_tower.deepstack_merger_list.2.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2668531998, "length_bytes": 4096 }, { "name": "vision_tower.deepstack_merger_list.2.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 2048, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3828208350, "length_bytes": 18874368 }, { "name": "vision_tower.deepstack_merger_list.2.norm.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3042241566, "length_bytes": 9216 }, { "name": "vision_tower.deepstack_merger_list.2.norm.weight", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3042250782, "length_bytes": 9216 }, { "name": "vision_tower.merger.linear_fc1.bias", "role": "other", "dtype": "bf16", "shape": [ 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4456173278, "length_bytes": 9216 }, { "name": "vision_tower.merger.linear_fc1.weight", "role": "other", "dtype": "bf16", "shape": [ 4608, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3148455710, "length_bytes": 42467328 }, { "name": "vision_tower.merger.linear_fc2.bias", "role": "other", "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4157714366, "length_bytes": 4096 }, { "name": "vision_tower.merger.linear_fc2.weight", "role": "other", "dtype": "bf16", "shape": [ 2048, 4608 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3110442526, "length_bytes": 18874368 }, { "name": "vision_tower.merger.norm.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190923038, "length_bytes": 2304 }, { "name": "vision_tower.merger.norm.weight", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229762654, "length_bytes": 2304 }, { "name": "vision_tower.patch_embed.proj.bias", "role": "other", "dtype": "bf16", "shape": [ 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4266362206, "length_bytes": 2304 }, { "name": "vision_tower.patch_embed.proj.weight", "role": "other", "dtype": "bf16", "shape": [ 1152, 2, 16, 16, 3 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5292111614, "length_bytes": 3538944 }, { "name": "vision_tower.pos_embed.weight", "role": "other", "dtype": "bf16", "shape": [ 2304, 1152 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3954207262, "length_bytes": 5308416 }, { "name": "language_model.model.embed_tokens.weight", "role": "token_embedding", "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 151936, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2682315294, "length_bytes": 311164928 }, { "name": "language_model.model.layers.0.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252870398, "length_bytes": 786432 }, { "name": "language_model.model.layers.0.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2668802334, "length_bytes": 256 }, { "name": "language_model.model.layers.0.input_layernorm.weight", "role": "attention_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2668798238, "length_bytes": 4096 }, { "name": "language_model.model.layers.0.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4888902654, "length_bytes": 6291456 }, { "name": "language_model.model.layers.0.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3228667422, "length_bytes": 4096 }, { "name": "language_model.model.layers.0.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4620266622, "length_bytes": 6291456 }, { "name": "language_model.model.layers.0.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 0, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2993480222, "length_bytes": 256 }, { "name": "language_model.model.layers.0.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2668802590, "length_bytes": 786432 }, { "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2441497886, "length_bytes": 100663296 }, { "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3962707998, "length_bytes": 100663296 }, { "name": "language_model.model.layers.0.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2668536094, "length_bytes": 262144 }, { "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 0, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2567868702, "length_bytes": 100663296 }, { "name": "language_model.model.layers.1.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3228676126, "length_bytes": 786432 }, { "name": "language_model.model.layers.1.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2426940446, "length_bytes": 256 }, { "name": "language_model.model.layers.1.input_layernorm.weight", "role": "attention_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2669593118, "length_bytes": 4096 }, { "name": "language_model.model.layers.1.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4753072830, "length_bytes": 6291456 }, { "name": "language_model.model.layers.1.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2426934046, "length_bytes": 4096 }, { "name": "language_model.model.layers.1.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2993482782, "length_bytes": 6291456 }, { "name": "language_model.model.layers.1.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 1, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3042784286, "length_bytes": 256 }, { "name": "language_model.model.layers.1.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2427079966, "length_bytes": 786432 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2300302110, "length_bytes": 100663296 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4759364286, "length_bytes": 100663296 }, { "name": "language_model.model.layers.1.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3135608350, "length_bytes": 262144 }, { "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 1, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4519589502, "length_bytes": 100663296 }, { "name": "language_model.model.layers.2.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2267525662, "length_bytes": 786432 }, { "name": "language_model.model.layers.2.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2259726622, "length_bytes": 256 }, { "name": "language_model.model.layers.2.input_layernorm.weight", "role": "attention_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5040694206, "length_bytes": 4096 }, { "name": "language_model.model.layers.2.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5330353406, "length_bytes": 6291456 }, { "name": "language_model.model.layers.2.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2247139614, "length_bytes": 4096 }, { "name": "language_model.model.layers.2.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2268844830, "length_bytes": 6291456 }, { "name": "language_model.model.layers.2.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 2, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4414691742, "length_bytes": 256 }, { "name": "language_model.model.layers.2.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2266673694, "length_bytes": 786432 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3239827966, "length_bytes": 100663296 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2146476318, "length_bytes": 100663296 }, { "name": "language_model.model.layers.2.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5330091262, "length_bytes": 262144 }, { "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 2, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4314028446, "length_bytes": 100663296 }, { "name": "language_model.model.layers.3.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5201864382, "length_bytes": 786432 }, { "name": "language_model.model.layers.3.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2106040094, "length_bytes": 256 }, { "name": "language_model.model.layers.3.input_layernorm.weight", "role": "attention_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5028089374, "length_bytes": 4096 }, { "name": "language_model.model.layers.3.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2260316702, "length_bytes": 6291456 }, { "name": "language_model.model.layers.3.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2106035998, "length_bytes": 4096 }, { "name": "language_model.model.layers.3.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3091552542, "length_bytes": 6291456 }, { "name": "language_model.model.layers.3.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 3, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2106040350, "length_bytes": 256 }, { "name": "language_model.model.layers.3.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2106761502, "length_bytes": 786432 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1766297374, "length_bytes": 100663296 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2005372702, "length_bytes": 100663296 }, { "name": "language_model.model.layers.3.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3814834398, "length_bytes": 262144 }, { "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 3, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1892126494, "length_bytes": 100663296 }, { "name": "language_model.model.layers.4.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1637846814, "length_bytes": 786432 }, { "name": "language_model.model.layers.4.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1528925470, "length_bytes": 256 }, { "name": "language_model.model.layers.4.input_layernorm.weight", "role": "attention_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1528921374, "length_bytes": 4096 }, { "name": "language_model.model.layers.4.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1530040094, "length_bytes": 6291456 }, { "name": "language_model.model.layers.4.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4424608414, "length_bytes": 4096 }, { "name": "language_model.model.layers.4.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3085251870, "length_bytes": 6291456 }, { "name": "language_model.model.layers.4.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 4, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1528925726, "length_bytes": 256 }, { "name": "language_model.model.layers.4.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1637060382, "length_bytes": 786432 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1075731742, "length_bytes": 100663296 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1315003678, "length_bytes": 100663296 }, { "name": "language_model.model.layers.4.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3927974430, "length_bytes": 262144 }, { "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 4, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1201691934, "length_bytes": 100663296 }, { "name": "language_model.model.layers.5.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1068653854, "length_bytes": 786432 }, { "name": "language_model.model.layers.5.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4142474686, "length_bytes": 256 }, { "name": "language_model.model.layers.5.input_layernorm.weight", "role": "attention_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 959466270, "length_bytes": 4096 }, { "name": "language_model.model.layers.5.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 960257054, "length_bytes": 6291456 }, { "name": "language_model.model.layers.5.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 959462174, "length_bytes": 4096 }, { "name": "language_model.model.layers.5.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1069440286, "length_bytes": 6291456 }, { "name": "language_model.model.layers.5.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 5, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 959470366, "length_bytes": 256 }, { "name": "language_model.model.layers.5.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1067342878, "length_bytes": 786432 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 720378654, "length_bytes": 100663296 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 858790686, "length_bytes": 100663296 }, { "name": "language_model.model.layers.5.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3928770078, "length_bytes": 262144 }, { "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 5, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1665634078, "length_bytes": 100663296 }, { "name": "language_model.model.layers.6.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 699931422, "length_bytes": 786432 }, { "name": "language_model.model.layers.6.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 692656670, "length_bytes": 256 }, { "name": "language_model.model.layers.6.input_layernorm.weight", "role": "attention_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252068350, "length_bytes": 4096 }, { "name": "language_model.model.layers.6.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 692656926, "length_bytes": 6291456 }, { "name": "language_model.model.layers.6.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3190936862, "length_bytes": 4096 }, { "name": "language_model.model.layers.6.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 701242142, "length_bytes": 6291456 }, { "name": "language_model.model.layers.6.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 6, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3393455774, "length_bytes": 256 }, { "name": "language_model.model.layers.6.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 699079454, "length_bytes": 786432 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4157720766, "length_bytes": 100663296 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3604512318, "length_bytes": 100663296 }, { "name": "language_model.model.layers.6.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 707533598, "length_bytes": 262144 }, { "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 6, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1415666974, "length_bytes": 100663296 }, { "name": "language_model.model.layers.7.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5252081662, "length_bytes": 786432 }, { "name": "language_model.model.layers.7.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1068653598, "length_bytes": 256 }, { "name": "language_model.model.layers.7.input_layernorm.weight", "role": "attention_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 646441502, "length_bytes": 4096 }, { "name": "language_model.model.layers.7.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 647494174, "length_bytes": 6291456 }, { "name": "language_model.model.layers.7.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 646437406, "length_bytes": 4096 }, { "name": "language_model.model.layers.7.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3129316894, "length_bytes": 6291456 }, { "name": "language_model.model.layers.7.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 7, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3925319966, "length_bytes": 256 }, { "name": "language_model.model.layers.7.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1664847646, "length_bytes": 786432 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 501733918, "length_bytes": 100663296 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 1536397086, "length_bytes": 100663296 }, { "name": "language_model.model.layers.7.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3229484990, "length_bytes": 262144 }, { "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 7, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5098539966, "length_bytes": 100663296 }, { "name": "language_model.model.layers.8.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 959470622, "length_bytes": 786432 }, { "name": "language_model.model.layers.8.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2268844574, "length_bytes": 256 }, { "name": "language_model.model.layers.8.input_layernorm.weight", "role": "attention_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3591937598, "length_bytes": 4096 }, { "name": "language_model.model.layers.8.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5215236958, "length_bytes": 6291456 }, { "name": "language_model.model.layers.8.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 469682462, "length_bytes": 4096 }, { "name": "language_model.model.layers.8.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 470276638, "length_bytes": 6291456 }, { "name": "language_model.model.layers.8.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 8, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 469686558, "length_bytes": 256 }, { "name": "language_model.model.layers.8.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2413548318, "length_bytes": 786432 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 966679582, "length_bytes": 100663296 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 368740638, "length_bytes": 100663296 }, { "name": "language_model.model.layers.8.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 469420318, "length_bytes": 262144 }, { "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 8, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 268077342, "length_bytes": 100663296 }, { "name": "language_model.model.layers.9.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5004193406, "length_bytes": 786432 }, { "name": "language_model.model.layers.9.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 241666078, "length_bytes": 256 }, { "name": "language_model.model.layers.9.input_layernorm.weight", "role": "attention_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2669589022, "length_bytes": 4096 }, { "name": "language_model.model.layers.9.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 640145950, "length_bytes": 6291456 }, { "name": "language_model.model.layers.9.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4511537118, "length_bytes": 4096 }, { "name": "language_model.model.layers.9.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4997893342, "length_bytes": 6291456 }, { "name": "language_model.model.layers.9.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 9, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4482666718, "length_bytes": 256 }, { "name": "language_model.model.layers.9.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4459367902, "length_bytes": 786432 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 14370846, "length_bytes": 100663296 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 140199966, "length_bytes": 100663296 }, { "name": "language_model.model.layers.9.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5344609790, "length_bytes": 262144 }, { "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 9, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3406035870, "length_bytes": 100663296 }, { "name": "language_model.model.layers.10.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5305566974, "length_bytes": 786432 }, { "name": "language_model.model.layers.10.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 83742, "length_bytes": 256 }, { "name": "language_model.model.layers.10.input_layernorm.weight", "role": "attention_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 2669662750, "length_bytes": 4096 }, { "name": "language_model.model.layers.10.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 4071333918, "length_bytes": 6291456 }, { "name": "language_model.model.layers.10.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 79646, "length_bytes": 4096 }, { "name": "language_model.model.layers.10.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 3795970014, "length_bytes": 6291456 }, { "name": "language_model.model.layers.10.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 10, "dtype": "bf16", "shape": [ 128 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5005524254, "length_bytes": 256 }, { "name": "language_model.model.layers.10.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 240879646, "length_bytes": 786432 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5194488177, "length_bytes": 100663296 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3237625201, "length_bytes": 100663296 }, { "name": "language_model.model.layers.10.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00001-of-00005.safetensors", "offset_bytes": 5344874238, "length_bytes": 262144 }, { "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 10, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4483245681, "length_bytes": 100663296 }, { "name": "language_model.model.layers.11.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2714168177, "length_bytes": 786432 }, { "name": "language_model.model.layers.11.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4763297393, "length_bytes": 256 }, { "name": "language_model.model.layers.11.input_layernorm.weight", "role": "attention_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4788869233, "length_bytes": 4096 }, { "name": "language_model.model.layers.11.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4037554801, "length_bytes": 6291456 }, { "name": "language_model.model.layers.11.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3485679217, "length_bytes": 4096 }, { "name": "language_model.model.layers.11.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3532247665, "length_bytes": 6291456 }, { "name": "language_model.model.layers.11.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 11, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4204180593, "length_bytes": 256 }, { "name": "language_model.model.layers.11.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4709738097, "length_bytes": 786432 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4609074801, "length_bytes": 100663296 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1912604529, "length_bytes": 100663296 }, { "name": "language_model.model.layers.11.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3465952625, "length_bytes": 262144 }, { "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 11, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3025407089, "length_bytes": 100663296 }, { "name": "language_model.model.layers.12.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4424447345, "length_bytes": 786432 }, { "name": "language_model.model.layers.12.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 12, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3498266225, "length_bytes": 256 }, { "name": "language_model.model.layers.12.input_layernorm.weight", "role": "attention_norm", "layer_index": 12, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4171133553, "length_bytes": 4096 }, { "name": "language_model.model.layers.12.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4031255153, "length_bytes": 6291456 }, { "name": "language_model.model.layers.12.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 12, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5181704305, "length_bytes": 4096 }, { "name": "language_model.model.layers.12.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4464367217, "length_bytes": 6291456 }, { "name": "language_model.model.layers.12.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 12, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4171149937, "length_bytes": 256 }, { "name": "language_model.model.layers.12.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4462990705, "length_bytes": 786432 }, { "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4814039153, "length_bytes": 100663296 }, { "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3779007089, "length_bytes": 100663296 }, { "name": "language_model.model.layers.12.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4157690481, "length_bytes": 262144 }, { "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 12, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4323784049, "length_bytes": 100663296 }, { "name": "language_model.model.layers.13.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3651850865, "length_bytes": 786432 }, { "name": "language_model.model.layers.13.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 13, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3511136113, "length_bytes": 256 }, { "name": "language_model.model.layers.13.input_layernorm.weight", "role": "attention_norm", "layer_index": 13, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3511132017, "length_bytes": 4096 }, { "name": "language_model.model.layers.13.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3127643505, "length_bytes": 6291456 }, { "name": "language_model.model.layers.13.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 13, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3237092721, "length_bytes": 4096 }, { "name": "language_model.model.layers.13.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3466804849, "length_bytes": 6291456 }, { "name": "language_model.model.layers.13.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 13, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2308073841, "length_bytes": 256 }, { "name": "language_model.model.layers.13.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2040801649, "length_bytes": 786432 }, { "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3538604657, "length_bytes": 100663296 }, { "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4915816561, "length_bytes": 100663296 }, { "name": "language_model.model.layers.13.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3510869873, "length_bytes": 262144 }, { "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 13, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2048993649, "length_bytes": 100663296 }, { "name": "language_model.model.layers.14.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3511136369, "length_bytes": 786432 }, { "name": "language_model.model.layers.14.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 14, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3126332529, "length_bytes": 256 }, { "name": "language_model.model.layers.14.input_layernorm.weight", "role": "attention_norm", "layer_index": 14, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4157952625, "length_bytes": 4096 }, { "name": "language_model.model.layers.14.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3525169521, "length_bytes": 6291456 }, { "name": "language_model.model.layers.14.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 14, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4171137649, "length_bytes": 4096 }, { "name": "language_model.model.layers.14.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5175412849, "length_bytes": 6291456 }, { "name": "language_model.model.layers.14.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 14, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4464301425, "length_bytes": 256 }, { "name": "language_model.model.layers.14.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3134590321, "length_bytes": 786432 }, { "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2556533361, "length_bytes": 100663296 }, { "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2714954609, "length_bytes": 100663296 }, { "name": "language_model.model.layers.14.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3126070385, "length_bytes": 262144 }, { "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 14, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3365289329, "length_bytes": 100663296 }, { "name": "language_model.model.layers.15.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2916346993, "length_bytes": 786432 }, { "name": "language_model.model.layers.15.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 15, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2815683441, "length_bytes": 256 }, { "name": "language_model.model.layers.15.input_layernorm.weight", "role": "attention_norm", "layer_index": 15, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2688662385, "length_bytes": 4096 }, { "name": "language_model.model.layers.15.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2301774193, "length_bytes": 6291456 }, { "name": "language_model.model.layers.15.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 15, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2308336241, "length_bytes": 4096 }, { "name": "language_model.model.layers.15.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2682370929, "length_bytes": 6291456 }, { "name": "language_model.model.layers.15.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 15, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2657196657, "length_bytes": 256 }, { "name": "language_model.model.layers.15.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2321517169, "length_bytes": 786432 }, { "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4204246385, "length_bytes": 100663296 }, { "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3917943409, "length_bytes": 100663296 }, { "name": "language_model.model.layers.15.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2308074097, "length_bytes": 262144 }, { "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 15, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2923957361, "length_bytes": 100663296 }, { "name": "language_model.model.layers.16.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2039490929, "length_bytes": 786432 }, { "name": "language_model.model.layers.16.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 16, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2026776433, "length_bytes": 256 }, { "name": "language_model.model.layers.16.input_layernorm.weight", "role": "attention_norm", "layer_index": 16, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2701314929, "length_bytes": 4096 }, { "name": "language_model.model.layers.16.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2033068401, "length_bytes": 6291456 }, { "name": "language_model.model.layers.16.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 16, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4470658673, "length_bytes": 4096 }, { "name": "language_model.model.layers.16.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2042702193, "length_bytes": 6291456 }, { "name": "language_model.model.layers.16.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 16, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2033068145, "length_bytes": 256 }, { "name": "language_model.model.layers.16.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3338288497, "length_bytes": 786432 }, { "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2187938161, "length_bytes": 100663296 }, { "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4043846257, "length_bytes": 100663296 }, { "name": "language_model.model.layers.16.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5136148593, "length_bytes": 262144 }, { "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 16, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3653169777, "length_bytes": 100663296 }, { "name": "language_model.model.layers.17.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4723631729, "length_bytes": 786432 }, { "name": "language_model.model.layers.17.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 17, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1728710257, "length_bytes": 256 }, { "name": "language_model.model.layers.17.input_layernorm.weight", "role": "attention_norm", "layer_index": 17, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4776282225, "length_bytes": 4096 }, { "name": "language_model.model.layers.17.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1728710513, "length_bytes": 6291456 }, { "name": "language_model.model.layers.17.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 17, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2320988785, "length_bytes": 4096 }, { "name": "language_model.model.layers.17.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5016545393, "length_bytes": 6291456 }, { "name": "language_model.model.layers.17.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 17, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3525169265, "length_bytes": 256 }, { "name": "language_model.model.layers.17.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1735001969, "length_bytes": 786432 }, { "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1501955697, "length_bytes": 100663296 }, { "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1628046961, "length_bytes": 100663296 }, { "name": "language_model.model.layers.17.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1627784817, "length_bytes": 262144 }, { "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 17, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2435549809, "length_bytes": 100663296 }, { "name": "language_model.model.layers.18.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1475479153, "length_bytes": 786432 }, { "name": "language_model.model.layers.18.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 18, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3466214769, "length_bytes": 256 }, { "name": "language_model.model.layers.18.input_layernorm.weight", "role": "attention_norm", "layer_index": 18, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4157956721, "length_bytes": 4096 }, { "name": "language_model.model.layers.18.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1468335729, "length_bytes": 6291456 }, { "name": "language_model.model.layers.18.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 18, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1454700145, "length_bytes": 4096 }, { "name": "language_model.model.layers.18.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4304909681, "length_bytes": 6291456 }, { "name": "language_model.model.layers.18.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 18, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4191073137, "length_bytes": 256 }, { "name": "language_model.model.layers.18.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1474692721, "length_bytes": 786432 }, { "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2334886513, "length_bytes": 100663296 }, { "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3135901041, "length_bytes": 100663296 }, { "name": "language_model.model.layers.18.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1453913713, "length_bytes": 262144 }, { "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 18, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1339618929, "length_bytes": 100663296 }, { "name": "language_model.model.layers.19.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1306850929, "length_bytes": 786432 }, { "name": "language_model.model.layers.19.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 19, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3532247409, "length_bytes": 256 }, { "name": "language_model.model.layers.19.input_layernorm.weight", "role": "attention_norm", "layer_index": 19, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1299039857, "length_bytes": 4096 }, { "name": "language_model.model.layers.19.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1299576433, "length_bytes": 6291456 }, { "name": "language_model.model.layers.19.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 19, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1299035761, "length_bytes": 4096 }, { "name": "language_model.model.layers.19.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1320744561, "length_bytes": 6291456 }, { "name": "language_model.model.layers.19.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 19, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5181708401, "length_bytes": 256 }, { "name": "language_model.model.layers.19.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1305998961, "length_bytes": 786432 }, { "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 971863409, "length_bytes": 100663296 }, { "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5022836849, "length_bytes": 100663296 }, { "name": "language_model.model.layers.19.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4763297649, "length_bytes": 262144 }, { "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 19, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1072526705, "length_bytes": 100663296 }, { "name": "language_model.model.layers.20.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2543164017, "length_bytes": 786432 }, { "name": "language_model.model.layers.20.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 20, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4776208241, "length_bytes": 256 }, { "name": "language_model.model.layers.20.input_layernorm.weight", "role": "attention_norm", "layer_index": 20, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2714164081, "length_bytes": 4096 }, { "name": "language_model.model.layers.20.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2917665905, "length_bytes": 6291456 }, { "name": "language_model.model.layers.20.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 20, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2543094385, "length_bytes": 4096 }, { "name": "language_model.model.layers.20.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 5295159665, "length_bytes": 6291456 }, { "name": "language_model.model.layers.20.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 20, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 968975473, "length_bytes": 256 }, { "name": "language_model.model.layers.20.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 969893233, "length_bytes": 786432 }, { "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 755000433, "length_bytes": 100663296 }, { "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 868312177, "length_bytes": 100663296 }, { "name": "language_model.model.layers.20.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2713901937, "length_bytes": 262144 }, { "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 20, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1173190001, "length_bytes": 100663296 }, { "name": "language_model.model.layers.21.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3024620657, "length_bytes": 786432 }, { "name": "language_model.model.layers.21.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 21, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 633496177, "length_bytes": 256 }, { "name": "language_model.model.layers.21.input_layernorm.weight", "role": "attention_norm", "layer_index": 21, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 627200625, "length_bytes": 4096 }, { "name": "language_model.model.layers.21.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2026776689, "length_bytes": 6291456 }, { "name": "language_model.model.layers.21.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 21, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3237621105, "length_bytes": 4096 }, { "name": "language_model.model.layers.21.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 635462769, "length_bytes": 6291456 }, { "name": "language_model.model.layers.21.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 21, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 633496433, "length_bytes": 256 }, { "name": "language_model.model.layers.21.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 634152049, "length_bytes": 786432 }, { "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 501363313, "length_bytes": 100663296 }, { "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2815683697, "length_bytes": 100663296 }, { "name": "language_model.model.layers.21.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 969565553, "length_bytes": 262144 }, { "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 21, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1774192497, "length_bytes": 100663296 }, { "name": "language_model.model.layers.22.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3126332785, "length_bytes": 786432 }, { "name": "language_model.model.layers.22.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 22, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 1286436209, "length_bytes": 256 }, { "name": "language_model.model.layers.22.input_layernorm.weight", "role": "attention_norm", "layer_index": 22, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3237088625, "length_bytes": 4096 }, { "name": "language_model.model.layers.22.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2536802929, "length_bytes": 6291456 }, { "name": "language_model.model.layers.22.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 22, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 970679665, "length_bytes": 4096 }, { "name": "language_model.model.layers.22.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 627204721, "length_bytes": 6291456 }, { "name": "language_model.model.layers.22.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 22, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 373035377, "length_bytes": 256 }, { "name": "language_model.model.layers.22.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 373625457, "length_bytes": 786432 }, { "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 45609329, "length_bytes": 100663296 }, { "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 272101745, "length_bytes": 100663296 }, { "name": "language_model.model.layers.22.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 372773233, "length_bytes": 262144 }, { "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 22, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 158855537, "length_bytes": 100663296 }, { "name": "language_model.model.layers.23.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3531460977, "length_bytes": 786432 }, { "name": "language_model.model.layers.23.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 23, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 13430641, "length_bytes": 256 }, { "name": "language_model.model.layers.23.input_layernorm.weight", "role": "attention_norm", "layer_index": 23, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 12902257, "length_bytes": 4096 }, { "name": "language_model.model.layers.23.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 4184781681, "length_bytes": 6291456 }, { "name": "language_model.model.layers.23.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 23, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 3510865777, "length_bytes": 4096 }, { "name": "language_model.model.layers.23.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 14152049, "length_bytes": 6291456 }, { "name": "language_model.model.layers.23.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 23, "dtype": "bf16", "shape": [ 128 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 13430897, "length_bytes": 256 }, { "name": "language_model.model.layers.23.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 2025916273, "length_bytes": 786432 }, { "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4713066847, "length_bytes": 100663296 }, { "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 654337137, "length_bytes": 100663296 }, { "name": "language_model.model.layers.23.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 12640113, "length_bytes": 262144 }, { "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 23, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00002-of-00005.safetensors", "offset_bytes": 374477425, "length_bytes": 100663296 }, { "name": "language_model.model.layers.24.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5142016607, "length_bytes": 786432 }, { "name": "language_model.model.layers.24.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 24, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4567244895, "length_bytes": 256 }, { "name": "language_model.model.layers.24.input_layernorm.weight", "role": "attention_norm", "layer_index": 24, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3867937887, "length_bytes": 4096 }, { "name": "language_model.model.layers.24.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4700414303, "length_bytes": 6291456 }, { "name": "language_model.model.layers.24.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 24, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1422778975, "length_bytes": 4096 }, { "name": "language_model.model.layers.24.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4011625823, "length_bytes": 6291456 }, { "name": "language_model.model.layers.24.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 24, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3611290207, "length_bytes": 256 }, { "name": "language_model.model.layers.24.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3893111903, "length_bytes": 786432 }, { "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4580876639, "length_bytes": 100663296 }, { "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4303003743, "length_bytes": 100663296 }, { "name": "language_model.model.layers.24.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4126248543, "length_bytes": 262144 }, { "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 24, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3168303455, "length_bytes": 100663296 }, { "name": "language_model.model.layers.25.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3439499871, "length_bytes": 786432 }, { "name": "language_model.model.layers.25.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 25, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4302802783, "length_bytes": 256 }, { "name": "language_model.model.layers.25.input_layernorm.weight", "role": "attention_norm", "layer_index": 25, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3902815327, "length_bytes": 4096 }, { "name": "language_model.model.layers.25.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4567245151, "length_bytes": 6291456 }, { "name": "language_model.model.layers.25.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 25, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3880524895, "length_bytes": 4096 }, { "name": "language_model.model.layers.25.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4574060895, "length_bytes": 6291456 }, { "name": "language_model.model.layers.25.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 25, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4011363423, "length_bytes": 256 }, { "name": "language_model.model.layers.25.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3167517023, "length_bytes": 786432 }, { "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4025585247, "length_bytes": 100663296 }, { "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3767274591, "length_bytes": 100663296 }, { "name": "language_model.model.layers.25.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4011363679, "length_bytes": 262144 }, { "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 25, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3338836575, "length_bytes": 100663296 }, { "name": "language_model.model.layers.26.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4010576991, "length_bytes": 786432 }, { "name": "language_model.model.layers.26.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 26, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2982623327, "length_bytes": 256 }, { "name": "language_model.model.layers.26.input_layernorm.weight", "role": "attention_norm", "layer_index": 26, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5156704863, "length_bytes": 4096 }, { "name": "language_model.model.layers.26.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3591551583, "length_bytes": 6291456 }, { "name": "language_model.model.layers.26.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 26, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3902811231, "length_bytes": 4096 }, { "name": "language_model.model.layers.26.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2982697311, "length_bytes": 6291456 }, { "name": "language_model.model.layers.26.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 26, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3294206303, "length_bytes": 256 }, { "name": "language_model.model.layers.26.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3165673823, "length_bytes": 786432 }, { "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3909831775, "length_bytes": 100663296 }, { "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1429140319, "length_bytes": 100663296 }, { "name": "language_model.model.layers.26.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3909438559, "length_bytes": 262144 }, { "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 26, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 768, 256 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3666611295, "length_bytes": 100663296 }, { "name": "language_model.model.layers.27.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3894422623, "length_bytes": 786432 }, { "name": "language_model.model.layers.27.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 27, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2012685407, "length_bytes": 256 }, { "name": "language_model.model.layers.27.input_layernorm.weight", "role": "attention_norm", "layer_index": 27, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5156696671, "length_bytes": 4096 }, { "name": "language_model.model.layers.27.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5353325407, "length_bytes": 6291456 }, { "name": "language_model.model.layers.27.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 27, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2975803487, "length_bytes": 4096 }, { "name": "language_model.model.layers.27.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3140507999, "length_bytes": 6291456 }, { "name": "language_model.model.layers.27.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 27, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1264959839, "length_bytes": 256 }, { "name": "language_model.model.layers.27.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4024798559, "length_bytes": 786432 }, { "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 4 }, "shape": [ 128, 2048, 96 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2704083039, "length_bytes": 100663296 }, { "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2824734815, "length_bytes": 150994944 }, { "name": "language_model.model.layers.27.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3909176415, "length_bytes": 262144 }, { "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 27, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2534213727, "length_bytes": 150994944 }, { "name": "language_model.model.layers.28.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2804811871, "length_bytes": 786432 }, { "name": "language_model.model.layers.28.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 28, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2352649567, "length_bytes": 256 }, { "name": "language_model.model.layers.28.input_layernorm.weight", "role": "attention_norm", "layer_index": 28, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2346354015, "length_bytes": 4096 }, { "name": "language_model.model.layers.28.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2505234527, "length_bytes": 6291456 }, { "name": "language_model.model.layers.28.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 28, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2346345823, "length_bytes": 4096 }, { "name": "language_model.model.layers.28.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2685208671, "length_bytes": 6291456 }, { "name": "language_model.model.layers.28.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 28, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2505226079, "length_bytes": 256 }, { "name": "language_model.model.layers.28.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2531199071, "length_bytes": 786432 }, { "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1688474719, "length_bytes": 150994944 }, { "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2031764831, "length_bytes": 150994944 }, { "name": "language_model.model.layers.28.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2805598303, "length_bytes": 262144 }, { "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 28, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2182759775, "length_bytes": 150994944 }, { "name": "language_model.model.layers.29.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3895209055, "length_bytes": 786432 }, { "name": "language_model.model.layers.29.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 29, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4025584991, "length_bytes": 256 }, { "name": "language_model.model.layers.29.input_layernorm.weight", "role": "attention_norm", "layer_index": 29, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1855394911, "length_bytes": 4096 }, { "name": "language_model.model.layers.29.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4972065119, "length_bytes": 6291456 }, { "name": "language_model.model.layers.29.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 29, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2531194975, "length_bytes": 4096 }, { "name": "language_model.model.layers.29.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2006393951, "length_bytes": 6291456 }, { "name": "language_model.model.layers.29.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 29, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1682117471, "length_bytes": 256 }, { "name": "language_model.model.layers.29.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1839469663, "length_bytes": 786432 }, { "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2352658271, "length_bytes": 150994944 }, { "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1530590047, "length_bytes": 150994944 }, { "name": "language_model.model.layers.29.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2524117087, "length_bytes": 262144 }, { "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 29, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1855399007, "length_bytes": 150994944 }, { "name": "language_model.model.layers.30.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1529803615, "length_bytes": 786432 }, { "name": "language_model.model.layers.30.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 30, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4126510687, "length_bytes": 256 }, { "name": "language_model.model.layers.30.input_layernorm.weight", "role": "attention_norm", "layer_index": 30, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4010568799, "length_bytes": 4096 }, { "name": "language_model.model.layers.30.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1422783071, "length_bytes": 6291456 }, { "name": "language_model.model.layers.30.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 30, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1263833439, "length_bytes": 4096 }, { "name": "language_model.model.layers.30.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3902884959, "length_bytes": 6291456 }, { "name": "language_model.model.layers.30.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 30, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3627019103, "length_bytes": 256 }, { "name": "language_model.model.layers.30.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3610425951, "length_bytes": 786432 }, { "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5188698975, "length_bytes": 150994944 }, { "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1264968287, "length_bytes": 150994944 }, { "name": "language_model.model.layers.30.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3591289439, "length_bytes": 262144 }, { "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 30, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4139093855, "length_bytes": 150994944 }, { "name": "language_model.model.layers.31.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1073778783, "length_bytes": 786432 }, { "name": "language_model.model.layers.31.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 31, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1087148127, "length_bytes": 256 }, { "name": "language_model.model.layers.31.input_layernorm.weight", "role": "attention_norm", "layer_index": 31, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3611286111, "length_bytes": 4096 }, { "name": "language_model.model.layers.31.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1067356255, "length_bytes": 6291456 }, { "name": "language_model.model.layers.31.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 31, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 915308383, "length_bytes": 4096 }, { "name": "language_model.model.layers.31.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1074565215, "length_bytes": 6291456 }, { "name": "language_model.model.layers.31.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 31, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1066307423, "length_bytes": 256 }, { "name": "language_model.model.layers.31.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5155385951, "length_bytes": 786432 }, { "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2989513055, "length_bytes": 150994944 }, { "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3440294495, "length_bytes": 150994944 }, { "name": "language_model.model.layers.31.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 915046239, "length_bytes": 262144 }, { "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 31, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 751386463, "length_bytes": 150994944 }, { "name": "language_model.model.layers.32.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1853494367, "length_bytes": 786432 }, { "name": "language_model.model.layers.32.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 32, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 710815071, "length_bytes": 256 }, { "name": "language_model.model.layers.32.input_layernorm.weight", "role": "attention_norm", "layer_index": 32, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 710810975, "length_bytes": 4096 }, { "name": "language_model.model.layers.32.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 711339871, "length_bytes": 6291456 }, { "name": "language_model.model.layers.32.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 32, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4706705759, "length_bytes": 4096 }, { "name": "language_model.model.layers.32.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 732249951, "length_bytes": 6291456 }, { "name": "language_model.model.layers.32.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 32, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 710815327, "length_bytes": 256 }, { "name": "language_model.model.layers.32.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 730283871, "length_bytes": 786432 }, { "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 382864735, "length_bytes": 150994944 }, { "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4416249951, "length_bytes": 150994944 }, { "name": "language_model.model.layers.32.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 710548831, "length_bytes": 262144 }, { "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 32, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 546442591, "length_bytes": 150994944 }, { "name": "language_model.model.layers.33.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 368971103, "length_bytes": 786432 }, { "name": "language_model.model.layers.33.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 33, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2352649823, "length_bytes": 256 }, { "name": "language_model.model.layers.33.input_layernorm.weight", "role": "attention_norm", "layer_index": 33, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 710544735, "length_bytes": 4096 }, { "name": "language_model.model.layers.33.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 723992415, "length_bytes": 6291456 }, { "name": "language_model.model.layers.33.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 33, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 723988319, "length_bytes": 4096 }, { "name": "language_model.model.layers.33.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 370281823, "length_bytes": 6291456 }, { "name": "language_model.model.layers.33.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 33, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 367594591, "length_bytes": 256 }, { "name": "language_model.model.layers.33.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 368184671, "length_bytes": 786432 }, { "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 203943007, "length_bytes": 150994944 }, { "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 915312479, "length_bytes": 150994944 }, { "name": "language_model.model.layers.33.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 744832863, "length_bytes": 262144 }, { "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 33, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4990947935, "length_bytes": 150994944 }, { "name": "language_model.model.layers.34.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 190573663, "length_bytes": 786432 }, { "name": "language_model.model.layers.34.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 34, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1429074527, "length_bytes": 256 }, { "name": "language_model.model.layers.34.input_layernorm.weight", "role": "attention_norm", "layer_index": 34, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2346349919, "length_bytes": 4096 }, { "name": "language_model.model.layers.34.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 184151135, "length_bytes": 6291456 }, { "name": "language_model.model.layers.34.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 34, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4302999647, "length_bytes": 4096 }, { "name": "language_model.model.layers.34.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 191360095, "length_bytes": 6291456 }, { "name": "language_model.model.layers.34.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 34, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5169816159, "length_bytes": 256 }, { "name": "language_model.model.layers.34.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 3611290463, "length_bytes": 786432 }, { "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4820545887, "length_bytes": 150994944 }, { "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 1112838495, "length_bytes": 150994944 }, { "name": "language_model.model.layers.34.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2504963935, "length_bytes": 262144 }, { "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 34, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 20556639, "length_bytes": 150994944 }, { "name": "language_model.model.layers.35.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 7187295, "length_bytes": 786432 }, { "name": "language_model.model.layers.35.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 35, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4990939487, "length_bytes": 256 }, { "name": "language_model.model.layers.35.input_layernorm.weight", "role": "attention_norm", "layer_index": 35, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4010572895, "length_bytes": 4096 }, { "name": "language_model.model.layers.35.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 830303, "length_bytes": 6291456 }, { "name": "language_model.model.layers.35.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 35, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 5156700767, "length_bytes": 4096 }, { "name": "language_model.model.layers.35.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 4814254431, "length_bytes": 6291456 }, { "name": "language_model.model.layers.35.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 35, "dtype": "bf16", "shape": [ 128 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 184150879, "length_bytes": 256 }, { "name": "language_model.model.layers.35.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 2504177503, "length_bytes": 786432 }, { "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3401388006, "length_bytes": 150994944 }, { "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4401120486, "length_bytes": 150994944 }, { "name": "language_model.model.layers.35.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00003-of-00005.safetensors", "offset_bytes": 43871, "length_bytes": 262144 }, { "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 35, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2746760166, "length_bytes": 150994944 }, { "name": "language_model.model.layers.36.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5231764966, "length_bytes": 786432 }, { "name": "language_model.model.layers.36.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 36, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3571667174, "length_bytes": 256 }, { "name": "language_model.model.layers.36.input_layernorm.weight", "role": "attention_norm", "layer_index": 36, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4558472678, "length_bytes": 4096 }, { "name": "language_model.model.layers.36.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2399292390, "length_bytes": 6291456 }, { "name": "language_model.model.layers.36.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 36, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5223896550, "length_bytes": 4096 }, { "name": "language_model.model.layers.36.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2589400038, "length_bytes": 6291456 }, { "name": "language_model.model.layers.36.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 36, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2905095398, "length_bytes": 256 }, { "name": "language_model.model.layers.36.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3586094310, "length_bytes": 786432 }, { "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1179092710, "length_bytes": 150994944 }, { "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2595765222, "length_bytes": 150994944 }, { "name": "language_model.model.layers.36.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4585018854, "length_bytes": 262144 }, { "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 36, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3587405030, "length_bytes": 150994944 }, { "name": "language_model.model.layers.37.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3585307878, "length_bytes": 786432 }, { "name": "language_model.model.layers.37.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 37, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3571667430, "length_bytes": 256 }, { "name": "language_model.model.layers.37.input_layernorm.weight", "role": "attention_norm", "layer_index": 37, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2213984742, "length_bytes": 4096 }, { "name": "language_model.model.layers.37.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3895751910, "length_bytes": 6291456 }, { "name": "language_model.model.layers.37.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 37, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3552382950, "length_bytes": 4096 }, { "name": "language_model.model.layers.37.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4204033254, "length_bytes": 6291456 }, { "name": "language_model.model.layers.37.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 37, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3571667686, "length_bytes": 256 }, { "name": "language_model.model.layers.37.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4920301030, "length_bytes": 786432 }, { "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5072082406, "length_bytes": 150994944 }, { "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3065278182, "length_bytes": 150994944 }, { "name": "language_model.model.layers.37.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3571405030, "length_bytes": 262144 }, { "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 37, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4592489958, "length_bytes": 150994944 }, { "name": "language_model.model.layers.38.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2905095654, "length_bytes": 786432 }, { "name": "language_model.model.layers.38.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 38, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3395096294, "length_bytes": 256 }, { "name": "language_model.model.layers.38.input_layernorm.weight", "role": "attention_norm", "layer_index": 38, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2595761126, "length_bytes": 4096 }, { "name": "language_model.model.layers.38.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2898803942, "length_bytes": 6291456 }, { "name": "language_model.model.layers.38.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 38, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3058450150, "length_bytes": 4096 }, { "name": "language_model.model.layers.38.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4901295590, "length_bytes": 6291456 }, { "name": "language_model.model.layers.38.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 38, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3585307622, "length_bytes": 256 }, { "name": "language_model.model.layers.38.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4571059686, "length_bytes": 786432 }, { "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2418174950, "length_bytes": 150994944 }, { "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3902043366, "length_bytes": 150994944 }, { "name": "language_model.model.layers.38.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5223634406, "length_bytes": 262144 }, { "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 38, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4749776358, "length_bytes": 150994944 }, { "name": "language_model.model.layers.39.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2398505702, "length_bytes": 786432 }, { "name": "language_model.model.layers.39.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 39, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5239956966, "length_bytes": 256 }, { "name": "language_model.model.layers.39.input_layernorm.weight", "role": "attention_norm", "layer_index": 39, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2377845222, "length_bytes": 4096 }, { "name": "language_model.model.layers.39.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2385189606, "length_bytes": 6291456 }, { "name": "language_model.model.layers.39.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 39, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4210324710, "length_bytes": 4096 }, { "name": "language_model.model.layers.39.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3558678758, "length_bytes": 6291456 }, { "name": "language_model.model.layers.39.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 39, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2378373606, "length_bytes": 256 }, { "name": "language_model.model.layers.39.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4591637990, "length_bytes": 786432 }, { "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1343785190, "length_bytes": 150994944 }, { "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3738465510, "length_bytes": 150994944 }, { "name": "language_model.model.layers.39.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2377583078, "length_bytes": 262144 }, { "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 39, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2220280294, "length_bytes": 150994944 }, { "name": "language_model.model.layers.40.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2206874086, "length_bytes": 786432 }, { "name": "language_model.model.layers.40.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 40, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3552387046, "length_bytes": 256 }, { "name": "language_model.model.layers.40.input_layernorm.weight", "role": "attention_norm", "layer_index": 40, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2199661030, "length_bytes": 4096 }, { "name": "language_model.model.layers.40.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3395096550, "length_bytes": 6291456 }, { "name": "language_model.model.layers.40.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 40, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3571392742, "length_bytes": 4096 }, { "name": "language_model.model.layers.40.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2371283430, "length_bytes": 6291456 }, { "name": "language_model.model.layers.40.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 40, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2898803686, "length_bytes": 256 }, { "name": "language_model.model.layers.40.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2206087654, "length_bytes": 786432 }, { "name": "language_model.model.layers.40.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1878378726, "length_bytes": 150994944 }, { "name": "language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2907455206, "length_bytes": 150994944 }, { "name": "language_model.model.layers.40.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1871300838, "length_bytes": 262144 }, { "name": "language_model.model.layers.40.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 40, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4053038310, "length_bytes": 150994944 }, { "name": "language_model.model.layers.41.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2588580838, "length_bytes": 786432 }, { "name": "language_model.model.layers.41.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 41, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4552115430, "length_bytes": 256 }, { "name": "language_model.model.layers.41.input_layernorm.weight", "role": "attention_norm", "layer_index": 41, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1698146534, "length_bytes": 4096 }, { "name": "language_model.model.layers.41.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1691789542, "length_bytes": 6291456 }, { "name": "language_model.model.layers.41.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 41, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1713486054, "length_bytes": 4096 }, { "name": "language_model.model.layers.41.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4367680742, "length_bytes": 6291456 }, { "name": "language_model.model.layers.41.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 41, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2199587046, "length_bytes": 256 }, { "name": "language_model.model.layers.41.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1698150630, "length_bytes": 786432 }, { "name": "language_model.model.layers.41.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1713490150, "length_bytes": 150994944 }, { "name": "language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3237801702, "length_bytes": 150994944 }, { "name": "language_model.model.layers.41.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1706932454, "length_bytes": 262144 }, { "name": "language_model.model.layers.41.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 41, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1521387750, "length_bytes": 150994944 }, { "name": "language_model.model.layers.42.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1501202662, "length_bytes": 786432 }, { "name": "language_model.model.layers.42.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 42, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1342736358, "length_bytes": 256 }, { "name": "language_model.model.layers.42.input_layernorm.weight", "role": "attention_norm", "layer_index": 42, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1179088614, "length_bytes": 4096 }, { "name": "language_model.model.layers.42.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1494780134, "length_bytes": 6291456 }, { "name": "language_model.model.layers.42.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 42, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2588576742, "length_bytes": 4096 }, { "name": "language_model.model.layers.42.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1502513382, "length_bytes": 6291456 }, { "name": "language_model.model.layers.42.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 42, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1336379110, "length_bytes": 256 }, { "name": "language_model.model.layers.42.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1706080486, "length_bytes": 786432 }, { "name": "language_model.model.layers.42.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 694572774, "length_bytes": 150994944 }, { "name": "language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1021728486, "length_bytes": 150994944 }, { "name": "language_model.model.layers.42.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4400334054, "length_bytes": 262144 }, { "name": "language_model.model.layers.42.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 42, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 858150630, "length_bytes": 150994944 }, { "name": "language_model.model.layers.43.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 512, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 1705228518, "length_bytes": 786432 }, { "name": "language_model.model.layers.43.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 43, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 672871654, "length_bytes": 256 }, { "name": "language_model.model.layers.43.input_layernorm.weight", "role": "attention_norm", "layer_index": 43, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 672867558, "length_bytes": 4096 }, { "name": "language_model.model.layers.43.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3552387302, "length_bytes": 6291456 }, { "name": "language_model.model.layers.43.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 43, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2398501606, "length_bytes": 4096 }, { "name": "language_model.model.layers.43.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 681989862, "length_bytes": 6291456 }, { "name": "language_model.model.layers.43.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 43, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 679163366, "length_bytes": 256 }, { "name": "language_model.model.layers.43.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 680285926, "length_bytes": 1048576 }, { "name": "language_model.model.layers.43.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 515310822, "length_bytes": 150994944 }, { "name": "language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4210328806, "length_bytes": 150994944 }, { "name": "language_model.model.layers.43.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 672605414, "length_bytes": 262144 }, { "name": "language_model.model.layers.43.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 43, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4921087462, "length_bytes": 150994944 }, { "name": "language_model.model.layers.44.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 5238908390, "length_bytes": 1048576 }, { "name": "language_model.model.layers.44.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 44, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2399292134, "length_bytes": 256 }, { "name": "language_model.model.layers.44.input_layernorm.weight", "role": "attention_norm", "layer_index": 44, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 487453670, "length_bytes": 4096 }, { "name": "language_model.model.layers.44.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2199730662, "length_bytes": 6291456 }, { "name": "language_model.model.layers.44.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 44, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2595757030, "length_bytes": 4096 }, { "name": "language_model.model.layers.44.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 496436454, "length_bytes": 6291456 }, { "name": "language_model.model.layers.44.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 44, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 487457766, "length_bytes": 256 }, { "name": "language_model.model.layers.44.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 488506598, "length_bytes": 1048576 }, { "name": "language_model.model.layers.44.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 172610534, "length_bytes": 150994944 }, { "name": "language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 336188390, "length_bytes": 150994944 }, { "name": "language_model.model.layers.44.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 487191526, "length_bytes": 262144 }, { "name": "language_model.model.layers.44.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 44, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2035992806, "length_bytes": 150994944 }, { "name": "language_model.model.layers.45.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 157930470, "length_bytes": 1048576 }, { "name": "language_model.model.layers.45.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 45, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2906406374, "length_bytes": 256 }, { "name": "language_model.model.layers.45.input_layernorm.weight", "role": "attention_norm", "layer_index": 45, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 157893606, "length_bytes": 4096 }, { "name": "language_model.model.layers.45.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 4552115686, "length_bytes": 6291456 }, { "name": "language_model.model.layers.45.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 45, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 157332198, "length_bytes": 4096 }, { "name": "language_model.model.layers.45.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 4096, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 160027622, "length_bytes": 6291456 }, { "name": "language_model.model.layers.45.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 45, "dtype": "bf16", "shape": [ 128 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 157336294, "length_bytes": 256 }, { "name": "language_model.model.layers.45.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 3222597350, "length_bytes": 1048576 }, { "name": "language_model.model.layers.45.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1661826360, "length_bytes": 150994944 }, { "name": "language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 6329062, "length_bytes": 150994944 }, { "name": "language_model.model.layers.45.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00004-of-00005.safetensors", "offset_bytes": 2029373670, "length_bytes": 262144 }, { "name": "language_model.model.layers.45.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 45, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 792490552, "length_bytes": 150994944 }, { "name": "language_model.model.layers.46.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1635345720, "length_bytes": 1048576 }, { "name": "language_model.model.layers.46.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 46, "dtype": "bf16", "shape": [ 128 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1297306168, "length_bytes": 256 }, { "name": "language_model.model.layers.46.input_layernorm.weight", "role": "attention_norm", "layer_index": 46, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1636394296, "length_bytes": 4096 }, { "name": "language_model.model.layers.46.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 6 }, "shape": [ 2048, 768 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1648981304, "length_bytes": 6291456 }, { "name": "language_model.model.layers.46.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 46, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1297285688, "length_bytes": 4096 }, { "name": "language_model.model.layers.46.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 4096, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1620665656, "length_bytes": 8388608 }, { "name": "language_model.model.layers.46.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 46, "dtype": "bf16", "shape": [ 128 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1303597880, "length_bytes": 256 }, { "name": "language_model.model.layers.46.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1619584312, "length_bytes": 1048576 }, { "name": "language_model.model.layers.46.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 955417144, "length_bytes": 150994944 }, { "name": "language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1139999288, "length_bytes": 150994944 }, { "name": "language_model.model.layers.46.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1619322168, "length_bytes": 262144 }, { "name": "language_model.model.layers.46.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 46, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1454593080, "length_bytes": 150994944 }, { "name": "language_model.model.layers.47.self_attn.k_proj.weight", "role": "attention_k", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 944632376, "length_bytes": 1048576 }, { "name": "language_model.model.layers.47.self_attn.k_norm.weight", "role": "attention_k_norm", "layer_index": 47, "dtype": "bf16", "shape": [ 128 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1605588024, "length_bytes": 256 }, { "name": "language_model.model.layers.47.input_layernorm.weight", "role": "attention_norm", "layer_index": 47, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 792486200, "length_bytes": 4096 }, { "name": "language_model.model.layers.47.self_attn.o_proj.weight", "role": "attention_o", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 2048, 1024 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1112703544, "length_bytes": 8388608 }, { "name": "language_model.model.layers.47.post_attention_layernorm.weight", "role": "attention_post_norm", "layer_index": 47, "dtype": "bf16", "shape": [ 2048 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1606128952, "length_bytes": 4096 }, { "name": "language_model.model.layers.47.self_attn.q_proj.weight", "role": "attention_q", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 4096, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 947028536, "length_bytes": 8388608 }, { "name": "language_model.model.layers.47.self_attn.q_norm.weight", "role": "attention_q_norm", "layer_index": 47, "dtype": "bf16", "shape": [ 128 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 792490296, "length_bytes": 256 }, { "name": "language_model.model.layers.47.self_attn.v_proj.weight", "role": "attention_v", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 8 }, "shape": [ 512, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 943518264, "length_bytes": 1048576 }, { "name": "language_model.model.layers.47.mlp.switch_mlp.down_proj.weight", "role": "ffn_down_exps", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 2048, 144 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 628629816, "length_bytes": 150994944 }, { "name": "language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight", "role": "ffn_gate_exps", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1303598136, "length_bytes": 150994944 }, { "name": "language_model.model.layers.47.mlp.gate.weight", "role": "ffn_gate_inp", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 32, "bits": 8 }, "shape": [ 128, 512 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 792224056, "length_bytes": 262144 }, { "name": "language_model.model.layers.47.mlp.switch_mlp.up_proj.weight", "role": "ffn_up_exps", "layer_index": 47, "dtype": "u32", "source_quantized": true, "quantization": { "mode": "affine", "group_size": 64, "bits": 6 }, "shape": [ 128, 768, 384 ], "file": "model-00005-of-00005.safetensors", "offset_bytes": 1812821304, "length_bytes": 150994944 } ] }