{ "estimated_full_resident_bytes": 226767375328, "estimated_max_layer_expert_bytes": 3422552064, "estimated_resident_bytes": 21414251488, "experts_per_tok": 10, "generated_by": "axquant", "mode": "layer-stack", "num_experts": 512, "required": false, "resident_roles": [ "embedding", "attention", "router", "shared_expert", "norm", "lm_head", "mtp" ], "schema_version": "axquant.expert-stream.v1", "streamed_roles": [ "expert" ], "tensors": [ { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00001-of-00046.safetensors", "group_size": 32, "layer": 0, "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 1, "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00002-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 2, "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00003-of-00046.safetensors", "group_size": 32, "layer": 3, "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00004-of-00046.safetensors", "group_size": 32, "layer": 4, "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 5, "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00005-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 6, "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00006-of-00046.safetensors", "group_size": 32, "layer": 7, "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00007-of-00046.safetensors", "group_size": 32, "layer": 8, "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 9, "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00008-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 10, "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00009-of-00046.safetensors", "group_size": 32, "layer": 11, "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00010-of-00046.safetensors", "group_size": 32, "layer": 12, "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 13, "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00011-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 14, "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00012-of-00046.safetensors", "group_size": 32, "layer": 15, "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00013-of-00046.safetensors", "group_size": 32, "layer": 16, "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 17, "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00014-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 18, "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00015-of-00046.safetensors", "group_size": 32, "layer": 19, "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00016-of-00046.safetensors", "group_size": 32, "layer": 20, "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 21, "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00017-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 22, "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00018-of-00046.safetensors", "group_size": 32, "layer": 23, "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00019-of-00046.safetensors", "group_size": 32, "layer": 24, "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 25, "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00020-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 26, "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00021-of-00046.safetensors", "group_size": 32, "layer": 27, "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00022-of-00046.safetensors", "group_size": 32, "layer": 28, "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 29, "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00023-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 30, "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00024-of-00046.safetensors", "group_size": 32, "layer": 31, "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00025-of-00046.safetensors", "group_size": 32, "layer": 32, "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 33, "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00026-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 34, "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00027-of-00046.safetensors", "group_size": 32, "layer": 35, "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00028-of-00046.safetensors", "group_size": 32, "layer": 36, "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 37, "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00029-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 38, "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00030-of-00046.safetensors", "group_size": 32, "layer": 39, "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00031-of-00046.safetensors", "group_size": 32, "layer": 40, "name": "language_model.model.layers.40.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 41, "name": "language_model.model.layers.41.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00032-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 42, "name": "language_model.model.layers.42.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00033-of-00046.safetensors", "group_size": 32, "layer": 43, "name": "language_model.model.layers.43.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00034-of-00046.safetensors", "group_size": 32, "layer": 44, "name": "language_model.model.layers.44.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 45, "name": "language_model.model.layers.45.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00035-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 46, "name": "language_model.model.layers.46.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00036-of-00046.safetensors", "group_size": 32, "layer": 47, "name": "language_model.model.layers.47.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00037-of-00046.safetensors", "group_size": 32, "layer": 48, "name": "language_model.model.layers.48.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 49, "name": "language_model.model.layers.49.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00038-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 50, "name": "language_model.model.layers.50.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00039-of-00046.safetensors", "group_size": 32, "layer": 51, "name": "language_model.model.layers.51.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00040-of-00046.safetensors", "group_size": 32, "layer": 52, "name": "language_model.model.layers.52.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 53, "name": "language_model.model.layers.53.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00041-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 54, "name": "language_model.model.layers.54.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00042-of-00046.safetensors", "group_size": 32, "layer": 55, "name": "language_model.model.layers.55.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00043-of-00046.safetensors", "group_size": 32, "layer": 56, "name": "language_model.model.layers.56.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 57, "name": "language_model.model.layers.57.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00044-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 58, "name": "language_model.model.layers.58.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00046-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.down_proj.scales", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00046-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.down_proj.weight", "num_experts": 512, "proj": "down" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.gate_proj.scales", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.gate_proj.weight", "num_experts": 512, "proj": "gate" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.up_proj.scales", "num_experts": 512, "proj": "up" }, { "bits": 4, "expert_axis": 0, "file": "model-00045-of-00046.safetensors", "group_size": 32, "layer": 59, "name": "language_model.model.layers.59.mlp.switch_mlp.up_proj.weight", "num_experts": 512, "proj": "up" } ] }