Upload folder using huggingface_hub

README.md

@@ -87,9 +87,6 @@ print(generated_text)
           "re:.*sconv$",
           "re:.*gate.*",
           "re:.*global_scale$",
-          "re:.*shared_experts.*",
-          "re:model\\.llm\\.layers\\.[01]\\.mlp\\..*",
-          "re:model\\.llm\\.layers\\.2\\.mlp\\.experts\\..*",
           "re:model\\.visual\\..*",
           "re:model\\.audio\\..*",
       ],

config.json

@@ -76,9 +76,6 @@
       "re:.*sconv$",
       "re:.*gate.*",
       "re:.*global_scale$",
-      "re:.*shared_experts.*",
-      "re:model\\.llm\\.layers\\.[01]\\.mlp\\..*",
-      "re:model\\.llm\\.layers\\.2\\.mlp\\.experts\\..*",
       "re:model\\.visual\\..*",
       "re:model\\.audio\\..*"
     ],

model-00001-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b85415cfa9ca085342fa75fae34ec19d2294d59ee727e0a456587c69b7578f27
-size 13023398844
+oid sha256:88c17b473bf4d8d7a9f5f8fe48a9b3dcf66e921159a02f0dd8bccffc5d1a5134
+size 8680230204

model-00002-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0699a085ac33070f2f3b4bd9c32ba26c6d59d2ce58206bb7fdfc31acd4c8aa92
-size 6549614336
+oid sha256:784ca81bff6169b945268a96dfbd718cbaecb8ac32ba2324fc9c4de12fbb9427
+size 6532853640

model-00004-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4ff08e9fc8ef503cf2be11f2afec3123733de21c92a63e0c11b6f0507ae7f9a
-size 8627684060
+oid sha256:96361adf7adba62e41233371d833a889c41e7a605c17646b1c7b1dc23280301b
+size 8610923356

model-00005-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da0fa8f09479abc0e807fc417ebff710bdbdf5723d34086194d1653ca9296072
-size 6501231500
+oid sha256:2ebfb910ed52b60a9448b84e77f412790239efccd97fab5eed13d26fd464fc2e
+size 6484470788

model-00006-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbf9ce2145196128b3cbe4cbf6e4487b4dddb90448488c70dfea352af9ad7651
-size 6985948500
+oid sha256:faaadfd8a9dceea4a354ee0179624809a73b2f1091ba58eaae56023b4d092c0e
+size 6851813380

model-00007-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fed67e7cfdebc4a3fb4ee8d1cf6c59f9e89c0c2a89d7eb94bbc62a8cac2c4f2e
-size 9005479422
+oid sha256:ebdc683c73bd46734374ebc050172e18865c9497432645c30ae14dce44da1ebe
+size 8904865702

model-00008-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb8f355f72142b7bab33425c68b68864f6b3e02f225bf04383527e6cfbe1bb5c
-size 6656503100
+oid sha256:5797d6c79dec5bdf3ff7cfa9ae5dd5586b9e330b1de1295d48b515296ad2b34e
+size 6572666548

model-00009-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21c7861ee44228325cc8e08649665f7b66ad26a2621d6341cfe190857877c95d
-size 9222481908
+oid sha256:bcd724f42fc4cd85a37e074b590009d749599de1118282527ec74581d5e81e4d
+size 9054808428

model-00010-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6255cf24c06c4346c02e63c4f1185b19145abc508cf85aad955e66b2db79b728
-size 6700652188
+oid sha256:cdaefae0573964c9bd1d9df5deb76398aa37288186c370618da9a5e979b5fa85
+size 6600013684

model-00011-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2104a8ea5c4f9f3caba6d10a5f61911f6bb2dceaed8191f795cb2a8a626be7b0
-size 8980210532
+oid sha256:fc7f6a8e01d8e8f78934956625f70eaf8e19716a30ece09979b4fef8dc478622
+size 8879613332

model-00012-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:863ac5a6a9a775edfef55c70225a223df853bbea037dd9e539a098ffea7e5b62
-size 9114458696
+oid sha256:10c852b8f99c0a6650e779c28c138804d5109193959de690ee50b7172506f0b8
+size 8980307072

model-00013-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97f11df0205eea0136e7785da1bdb018d6f41967c319f0304beafa7a0c053378
-size 9055604424
+oid sha256:f80e7b7d81d58b52c155b7aa280b4c708d8efe3f6c96bb45cc41c6c739de0bf3
+size 8904667320

model-00014-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc0f3acb709d22528132250df4bb3a4222968de69f8506446f83579a778d2ffa
-size 8946672208
+oid sha256:9b60bc0a0a6e5ab6080109d00df2cf0969d14709ac3508ce0cf206c6088e31c0
+size 8812537056

model-00015-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e14b63a71d153b4eacf5ef9aab0a90e25cd96b7725cd161bf0aaf52287b6478e
-size 6639628054
+oid sha256:6fd87e575c0ed0b7bde68e8bac9485103913e3895688f62da3e2be06414dbbae
+size 6555791526

model-00017-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61f9ddacd447428ee919309f31928d0a2fda03b84916e2618e4282bb24fc9bce
-size 11092055996
+oid sha256:925cd0ab87a64865159df34813c0a921994ab7279699fcf57f8083812b28d6d8
+size 8862849580

model-00018-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a384348ec54f8345c18420cba6dfbea5fe43c7b213e0a3893937e562314fbe94
-size 6878177172
+oid sha256:207da931397827ed12c85df0a5a4386bc1c73376a7d25ac935e6de858a1918d3
+size 6760802724

model-00020-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81e19918e259a446cf299c5514a602374841dab13b3c73fc524acc56b7cb61c3
-size 8745199788
+oid sha256:f93ae9cc0792e1c33869a1d0fe8693e0f4ed196b9c9a96f4db60fc8d478d259e
+size 8694901188

model-00021-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:457b39bc448e20056b0f40307565e8d657494cc6cbd9de5bee4a23222c776881
-size 9053614572
+oid sha256:9a7e03ffa28bb5e51fd22cd808a642bcd016016eba623fbd7d8a0d485541357e
+size 8936256668

model-00022-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7cbcffa99f45dd95c65a73e7c036bf6d88570dac98b70abd27042ea3b8cca45
-size 9313859904
+oid sha256:660cd6a19de8b498720929cd2cf00806d471436652e1d4020129bd9ece2995c7
+size 9129442696

model-00023-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c287f4b9df4130563b811ca254f52ebd817442c34e575970c5218d9baa52098
-size 8875308416
+oid sha256:82119b102212cddaff49e8b01f803c5d1a93228cd36c76bc370a2fba145b8958
+size 8757950496

model-00024-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6d6be24870d64b78fd1a1efc29f0925cdb695e2cb9af755fb3988f553820052
-size 8753803280
+oid sha256:e6798b7da78c7fb8f46b41adb356e17dfee68f1d8782b9c41073d11599eb9c91
+size 8686727424

model-00025-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:758f4c560f3bf2b613159a229d2197b6a45029a08b3f8520333b04e4925ede75
-size 8885735796
+oid sha256:d54f75267b9a9bd2d89dd85aba2e25999cfd368392b7aa21d9c820b7432cd5de
+size 8818676476

model-00026-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e189e39ccf62acdda3d97c5c77a2baa22dfb4f833567413c956de3f1dc9378d9
-size 9005501660
+oid sha256:e82adecebb76b28b0f452f323c258ebdb0b0901413993f3efb6d3e3023525d09
+size 8921681684

model-00027-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a16563788c55984e5ae658663bbfcfd3e2016016f73b68412c388fb982da34b
-size 8734798436
+oid sha256:0cd05fcd0a374094bd5cb9ee74a3d068e68e9d15c4fa5b3754cd1aa41bd6250a
+size 8701260516

model-00028-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8d6b0c3b61a1d3ce7d92f01fc4f869557a3f4535cffd268cbbe09e053cbab0d
-size 8847969784
+oid sha256:4bb0aaf26154d9f0c85d16c06ac21012ff79e672dc4b99b5b83ffafddd99124d
+size 8814448408

model-00029-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2121c7a72e6279ef28d210f3b45329835b7863b9710d8a1def14fdd8ca18ce1f
-size 8883687864
+oid sha256:3122c249c34f561ccd5920cd59402ef0b6e1c1c80942cf8c0e1414784671ecf8
+size 8749535728

model-00032-of-00032.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb9fc417552155dccfbf5f07185c54bae00ce8c2c38705bc97d853720da357eb
-size 8808166576
+oid sha256:d9cccc0de2a3fd614d00124ca7e0d69bcbf8e178cb5e35b611441ae792a5f21e
+size 8774645192

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 278942496948
+    "total_size": 270089779380
   },
   "weight_map": {
     "model.audio.encoder.weight": "model-00019-of-00032.safetensors",
@@ -21,7 +21,9 @@
     "model.llm.layers.0.attn_sconv.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.0.mlp.global_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.0.mlp.w13_dn.weight": "model-00009-of-00032.safetensors",
+    "model.llm.layers.0.mlp.w13_dn.weight_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.0.mlp.w2_md.weight": "model-00013-of-00032.safetensors",
+    "model.llm.layers.0.mlp.w2_md.weight_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.0.mlp_norm.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.0.mlp_sconv.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.1.attn.k_norm.weight": "model-00011-of-00032.safetensors",
@@ -38,7 +40,9 @@
     "model.llm.layers.1.attn_sconv.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.1.mlp.global_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.1.mlp.w13_dn.weight": "model-00029-of-00032.safetensors",
+    "model.llm.layers.1.mlp.w13_dn.weight_scale": "model-00029-of-00032.safetensors",
     "model.llm.layers.1.mlp.w2_md.weight": "model-00010-of-00032.safetensors",
+    "model.llm.layers.1.mlp.w2_md.weight_scale": "model-00010-of-00032.safetensors",
     "model.llm.layers.1.mlp_norm.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.1.mlp_sconv.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.10.attn.k_norm.weight": "model-00013-of-00032.safetensors",
@@ -61,7 +65,9 @@
     "model.llm.layers.10.mlp.gate.global_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.10.mlp.gate.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.10.mlp.shared_experts.shared_w13_weight": "model-00023-of-00032.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w13_weight_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.10.mlp.shared_experts.shared_w2_weight": "model-00021-of-00032.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w2_weight_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.10.mlp_norm.weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.10.mlp_sconv.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.11.attn.k_norm.weight": "model-00021-of-00032.safetensors",
@@ -84,7 +90,9 @@
     "model.llm.layers.11.mlp.gate.global_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.11.mlp.gate.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.11.mlp.shared_experts.shared_w13_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w13_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.11.mlp.shared_experts.shared_w2_weight": "model-00004-of-00032.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w2_weight_scale": "model-00004-of-00032.safetensors",
     "model.llm.layers.11.mlp_norm.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.11.mlp_sconv.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.12.attn.k_norm.weight": "model-00029-of-00032.safetensors",
@@ -107,7 +115,9 @@
     "model.llm.layers.12.mlp.gate.global_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.12.mlp.gate.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.12.mlp.shared_experts.shared_w13_weight": "model-00015-of-00032.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w13_weight_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.12.mlp.shared_experts.shared_w2_weight": "model-00026-of-00032.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w2_weight_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.12.mlp_norm.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.12.mlp_sconv.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.13.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -130,7 +140,9 @@
     "model.llm.layers.13.mlp.gate.global_scale": "model-00004-of-00032.safetensors",
     "model.llm.layers.13.mlp.gate.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.13.mlp.shared_experts.shared_w13_weight": "model-00014-of-00032.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w13_weight_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.13.mlp.shared_experts.shared_w2_weight": "model-00005-of-00032.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w2_weight_scale": "model-00005-of-00032.safetensors",
     "model.llm.layers.13.mlp_norm.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.13.mlp_sconv.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.14.attn.k_norm.weight": "model-00019-of-00032.safetensors",
@@ -153,7 +165,9 @@
     "model.llm.layers.14.mlp.gate.global_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.14.mlp.gate.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.14.mlp.shared_experts.shared_w13_weight": "model-00008-of-00032.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w13_weight_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.14.mlp.shared_experts.shared_w2_weight": "model-00011-of-00032.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w2_weight_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.14.mlp_norm.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.14.mlp_sconv.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.15.attn.k_norm.weight": "model-00021-of-00032.safetensors",
@@ -176,7 +190,9 @@
     "model.llm.layers.15.mlp.gate.global_scale": "model-00020-of-00032.safetensors",
     "model.llm.layers.15.mlp.gate.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.15.mlp.shared_experts.shared_w13_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w13_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.15.mlp.shared_experts.shared_w2_weight": "model-00009-of-00032.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w2_weight_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.15.mlp_norm.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.15.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.16.attn.k_norm.weight": "model-00028-of-00032.safetensors",
@@ -199,7 +215,9 @@
     "model.llm.layers.16.mlp.gate.global_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.16.mlp.gate.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.16.mlp.shared_experts.shared_w13_weight": "model-00014-of-00032.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w13_weight_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.16.mlp.shared_experts.shared_w2_weight": "model-00008-of-00032.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w2_weight_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.16.mlp_norm.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.16.mlp_sconv.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.17.attn.k_norm.weight": "model-00009-of-00032.safetensors",
@@ -222,7 +240,9 @@
     "model.llm.layers.17.mlp.gate.global_scale": "model-00031-of-00032.safetensors",
     "model.llm.layers.17.mlp.gate.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.17.mlp.shared_experts.shared_w13_weight": "model-00007-of-00032.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w13_weight_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.17.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w2_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.17.mlp_norm.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.17.mlp_sconv.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.18.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -245,7 +265,9 @@
     "model.llm.layers.18.mlp.gate.global_scale": "model-00027-of-00032.safetensors",
     "model.llm.layers.18.mlp.gate.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.18.mlp.shared_experts.shared_w13_weight": "model-00027-of-00032.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w13_weight_scale": "model-00027-of-00032.safetensors",
     "model.llm.layers.18.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w2_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.18.mlp_norm.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.18.mlp_sconv.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.19.attn.k_norm.weight": "model-00018-of-00032.safetensors",
@@ -268,7 +290,9 @@
     "model.llm.layers.19.mlp.gate.global_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.19.mlp.gate.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.19.mlp.shared_experts.shared_w13_weight": "model-00025-of-00032.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w13_weight_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.19.mlp.shared_experts.shared_w2_weight": "model-00025-of-00032.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w2_weight_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.19.mlp_norm.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.19.mlp_sconv.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.2.attn.k_norm.weight": "model-00025-of-00032.safetensors",
@@ -284,12 +308,16 @@
     "model.llm.layers.2.attn_norm.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.2.attn_sconv.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.2.mlp.experts.w13_weight": "model-00001-of-00032.safetensors",
+    "model.llm.layers.2.mlp.experts.w13_weight_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.2.mlp.experts.w2_weight": "model-00017-of-00032.safetensors",
+    "model.llm.layers.2.mlp.experts.w2_weight_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.2.mlp.gate.bias": "model-00001-of-00032.safetensors",
     "model.llm.layers.2.mlp.gate.global_scale": "model-00003-of-00032.safetensors",
     "model.llm.layers.2.mlp.gate.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.2.mlp.shared_experts.shared_w13_weight": "model-00001-of-00032.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w13_weight_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.2.mlp.shared_experts.shared_w2_weight": "model-00025-of-00032.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w2_weight_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.2.mlp_norm.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.2.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.20.attn.k_norm.weight": "model-00003-of-00032.safetensors",
@@ -312,7 +340,9 @@
     "model.llm.layers.20.mlp.gate.global_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.20.mlp.gate.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.20.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w13_weight_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.20.mlp.shared_experts.shared_w2_weight": "model-00013-of-00032.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w2_weight_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.20.mlp_norm.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.20.mlp_sconv.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.21.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -335,7 +365,9 @@
     "model.llm.layers.21.mlp.gate.global_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.21.mlp.gate.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.21.mlp.shared_experts.shared_w13_weight": "model-00024-of-00032.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w13_weight_scale": "model-00024-of-00032.safetensors",
     "model.llm.layers.21.mlp.shared_experts.shared_w2_weight": "model-00006-of-00032.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w2_weight_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.21.mlp_norm.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.21.mlp_sconv.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.22.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -358,7 +390,9 @@
     "model.llm.layers.22.mlp.gate.global_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.22.mlp.gate.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.22.mlp.shared_experts.shared_w13_weight": "model-00018-of-00032.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w13_weight_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.22.mlp.shared_experts.shared_w2_weight": "model-00001-of-00032.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w2_weight_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.22.mlp_norm.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.22.mlp_sconv.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.23.attn.k_norm.weight": "model-00017-of-00032.safetensors",
@@ -381,7 +415,9 @@
     "model.llm.layers.23.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.23.mlp.gate.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.23.mlp.shared_experts.shared_w13_weight": "model-00006-of-00032.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w13_weight_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.23.mlp.shared_experts.shared_w2_weight": "model-00023-of-00032.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w2_weight_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.23.mlp_norm.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.23.mlp_sconv.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.24.attn.k_norm.weight": "model-00029-of-00032.safetensors",
@@ -404,7 +440,9 @@
     "model.llm.layers.24.mlp.gate.global_scale": "model-00010-of-00032.safetensors",
     "model.llm.layers.24.mlp.gate.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.24.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w13_weight_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.24.mlp.shared_experts.shared_w2_weight": "model-00018-of-00032.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w2_weight_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.24.mlp_norm.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.24.mlp_sconv.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.25.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -427,7 +465,9 @@
     "model.llm.layers.25.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.25.mlp.gate.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.25.mlp.shared_experts.shared_w13_weight": "model-00011-of-00032.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w13_weight_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.25.mlp.shared_experts.shared_w2_weight": "model-00017-of-00032.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w2_weight_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.25.mlp_norm.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.25.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.26.attn.k_norm.weight": "model-00018-of-00032.safetensors",
@@ -450,7 +490,9 @@
     "model.llm.layers.26.mlp.gate.global_scale": "model-00002-of-00032.safetensors",
     "model.llm.layers.26.mlp.gate.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.mlp.shared_experts.shared_w13_weight": "model-00013-of-00032.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w13_weight_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.26.mlp.shared_experts.shared_w2_weight": "model-00023-of-00032.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w2_weight_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.26.mlp_norm.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.26.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.27.attn.k_norm.weight": "model-00003-of-00032.safetensors",
@@ -473,7 +515,9 @@
     "model.llm.layers.27.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.27.mlp.gate.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.27.mlp.shared_experts.shared_w13_weight": "model-00018-of-00032.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w13_weight_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.27.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w2_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.27.mlp_norm.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.27.mlp_sconv.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.28.attn.k_norm.weight": "model-00012-of-00032.safetensors",
@@ -496,7 +540,9 @@
     "model.llm.layers.28.mlp.gate.global_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.28.mlp.gate.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.28.mlp.shared_experts.shared_w13_weight": "model-00026-of-00032.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w13_weight_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.28.mlp.shared_experts.shared_w2_weight": "model-00021-of-00032.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w2_weight_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.28.mlp_norm.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.28.mlp_sconv.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.29.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -519,7 +565,9 @@
     "model.llm.layers.29.mlp.gate.global_scale": "model-00002-of-00032.safetensors",
     "model.llm.layers.29.mlp.gate.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.29.mlp.shared_experts.shared_w13_weight": "model-00024-of-00032.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w13_weight_scale": "model-00024-of-00032.safetensors",
     "model.llm.layers.29.mlp.shared_experts.shared_w2_weight": "model-00015-of-00032.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w2_weight_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.29.mlp_norm.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.29.mlp_sconv.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.3.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -542,7 +590,9 @@
     "model.llm.layers.3.mlp.gate.global_scale": "model-00016-of-00032.safetensors",
     "model.llm.layers.3.mlp.gate.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.3.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w13_weight_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.3.mlp.shared_experts.shared_w2_weight": "model-00011-of-00032.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w2_weight_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.mlp_norm.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.3.mlp_sconv.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.30.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -565,7 +615,9 @@
     "model.llm.layers.30.mlp.gate.global_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.30.mlp.gate.weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.30.mlp.shared_experts.shared_w13_weight": "model-00007-of-00032.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w13_weight_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.30.mlp.shared_experts.shared_w2_weight": "model-00002-of-00032.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w2_weight_scale": "model-00002-of-00032.safetensors",
     "model.llm.layers.30.mlp_norm.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.30.mlp_sconv.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.31.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -588,7 +640,9 @@
     "model.llm.layers.31.mlp.gate.global_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.31.mlp.gate.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.31.mlp.shared_experts.shared_w13_weight": "model-00017-of-00032.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w13_weight_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.31.mlp.shared_experts.shared_w2_weight": "model-00032-of-00032.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w2_weight_scale": "model-00032-of-00032.safetensors",
     "model.llm.layers.31.mlp_norm.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.31.mlp_sconv.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.32.attn.k_norm.weight": "model-00023-of-00032.safetensors",
@@ -611,7 +665,9 @@
     "model.llm.layers.32.mlp.gate.global_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.32.mlp.gate.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.32.mlp.shared_experts.shared_w13_weight": "model-00015-of-00032.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w13_weight_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.32.mlp.shared_experts.shared_w2_weight": "model-00014-of-00032.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w2_weight_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.32.mlp_norm.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.32.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.33.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -634,7 +690,9 @@
     "model.llm.layers.33.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.33.mlp.gate.weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.mlp.shared_experts.shared_w13_weight": "model-00021-of-00032.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w13_weight_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.33.mlp.shared_experts.shared_w2_weight": "model-00020-of-00032.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w2_weight_scale": "model-00020-of-00032.safetensors",
     "model.llm.layers.33.mlp_norm.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.33.mlp_sconv.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.34.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -657,7 +715,9 @@
     "model.llm.layers.34.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.34.mlp.gate.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.34.mlp.shared_experts.shared_w13_weight": "model-00006-of-00032.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w13_weight_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.34.mlp.shared_experts.shared_w2_weight": "model-00009-of-00032.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w2_weight_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.34.mlp_norm.weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.34.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.35.attn.k_norm.weight": "model-00021-of-00032.safetensors",
@@ -680,7 +740,9 @@
     "model.llm.layers.35.mlp.gate.global_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.35.mlp.gate.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.35.mlp.shared_experts.shared_w13_weight": "model-00021-of-00032.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w13_weight_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.35.mlp.shared_experts.shared_w2_weight": "model-00014-of-00032.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w2_weight_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.mlp_norm.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.35.mlp_sconv.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.36.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -703,7 +765,9 @@
     "model.llm.layers.36.mlp.gate.global_scale": "model-00003-of-00032.safetensors",
     "model.llm.layers.36.mlp.gate.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.36.mlp.shared_experts.shared_w13_weight": "model-00007-of-00032.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w13_weight_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.36.mlp.shared_experts.shared_w2_weight": "model-00028-of-00032.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w2_weight_scale": "model-00028-of-00032.safetensors",
     "model.llm.layers.36.mlp_norm.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.36.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.37.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -726,7 +790,9 @@
     "model.llm.layers.37.mlp.gate.global_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.37.mlp.gate.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.37.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w13_weight_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.37.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w2_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.37.mlp_norm.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.37.mlp_sconv.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.38.attn.k_norm.weight": "model-00007-of-00032.safetensors",
@@ -749,7 +815,9 @@
     "model.llm.layers.38.mlp.gate.global_scale": "model-00019-of-00032.safetensors",
     "model.llm.layers.38.mlp.gate.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.38.mlp.shared_experts.shared_w13_weight": "model-00006-of-00032.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w13_weight_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.38.mlp.shared_experts.shared_w2_weight": "model-00017-of-00032.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w2_weight_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.38.mlp_norm.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.38.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.39.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -772,7 +840,9 @@
     "model.llm.layers.39.mlp.gate.global_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.39.mlp.gate.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.39.mlp.shared_experts.shared_w13_weight": "model-00008-of-00032.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w13_weight_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.39.mlp.shared_experts.shared_w2_weight": "model-00026-of-00032.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w2_weight_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.39.mlp_norm.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.39.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.4.attn.k_norm.weight": "model-00029-of-00032.safetensors",
@@ -795,7 +865,9 @@
     "model.llm.layers.4.mlp.gate.global_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.4.mlp.gate.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.4.mlp.shared_experts.shared_w13_weight": "model-00011-of-00032.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w13_weight_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.4.mlp.shared_experts.shared_w2_weight": "model-00006-of-00032.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w2_weight_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.4.mlp_norm.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.4.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.40.attn.k_norm.weight": "model-00011-of-00032.safetensors",
@@ -818,7 +890,9 @@
     "model.llm.layers.40.mlp.gate.global_scale": "model-00005-of-00032.safetensors",
     "model.llm.layers.40.mlp.gate.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.40.mlp.shared_experts.shared_w13_weight": "model-00013-of-00032.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w13_weight_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.40.mlp.shared_experts.shared_w2_weight": "model-00032-of-00032.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w2_weight_scale": "model-00032-of-00032.safetensors",
     "model.llm.layers.40.mlp_norm.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.40.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.41.attn.k_norm.weight": "model-00005-of-00032.safetensors",
@@ -841,7 +915,9 @@
     "model.llm.layers.41.mlp.gate.global_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp.gate.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.41.mlp.shared_experts.shared_w13_weight": "model-00010-of-00032.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w13_weight_scale": "model-00010-of-00032.safetensors",
     "model.llm.layers.41.mlp.shared_experts.shared_w2_weight": "model-00021-of-00032.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w2_weight_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.41.mlp_norm.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp_sconv.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.5.attn.k_norm.weight": "model-00020-of-00032.safetensors",
@@ -864,7 +940,9 @@
     "model.llm.layers.5.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp.gate.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.5.mlp.shared_experts.shared_w13_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w13_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w2_weight_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp_norm.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.5.mlp_sconv.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.6.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -887,7 +965,9 @@
     "model.llm.layers.6.mlp.gate.global_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.6.mlp.gate.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.6.mlp.shared_experts.shared_w13_weight": "model-00014-of-00032.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w13_weight_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.6.mlp.shared_experts.shared_w2_weight": "model-00028-of-00032.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w2_weight_scale": "model-00028-of-00032.safetensors",
     "model.llm.layers.6.mlp_norm.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.6.mlp_sconv.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.7.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -910,7 +990,9 @@
     "model.llm.layers.7.mlp.gate.global_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.7.mlp.gate.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.7.mlp.shared_experts.shared_w13_weight": "model-00020-of-00032.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w13_weight_scale": "model-00020-of-00032.safetensors",
     "model.llm.layers.7.mlp.shared_experts.shared_w2_weight": "model-00023-of-00032.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w2_weight_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.7.mlp_norm.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.7.mlp_sconv.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.8.attn.k_norm.weight": "model-00011-of-00032.safetensors",
@@ -933,7 +1015,9 @@
     "model.llm.layers.8.mlp.gate.global_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.8.mlp.gate.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.8.mlp.shared_experts.shared_w13_weight": "model-00023-of-00032.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w13_weight_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.8.mlp.shared_experts.shared_w2_weight": "model-00026-of-00032.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w2_weight_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.8.mlp_norm.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.8.mlp_sconv.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.9.attn.k_norm.weight": "model-00018-of-00032.safetensors",
@@ -956,7 +1040,9 @@
     "model.llm.layers.9.mlp.gate.global_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.9.mlp.gate.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.9.mlp.shared_experts.shared_w13_weight": "model-00018-of-00032.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w13_weight_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.9.mlp.shared_experts.shared_w2_weight": "model-00017-of-00032.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w2_weight_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.9.mlp_norm.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.9.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.norm.weight": "model-00025-of-00032.safetensors",