model.embed_tokens.weight F32 256x64 65536 55d6c0ed1919ad2e4048cb4e177ddc46a99986753226f68088ce9efdff9b12b2 model.layers.0.input_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.0.mlp.down_proj.weight F32 64x128 32768 37920cb067b1037df8e13a9af9691a5dfcafb215849a8be860790effe445d341 model.layers.0.mlp.gate_proj.weight F32 128x64 32768 db239d0b3b286292442a4de0dd2d185a7fea4264b0da30f93efd16b5210bb1fd model.layers.0.mlp.up_proj.weight F32 128x64 32768 5b8ab821033a6bfd653da02c8f6c4e15a4e41ea683c6a906f81557cf345841f8 model.layers.0.post_attention_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.0.self_attn.k_proj.weight F32 64x64 16384 ede9fea178bbefaea20b3b7f88bee6ce393a3949635ca43c3010b2891fdacf8e model.layers.0.self_attn.o_proj.weight F32 64x64 16384 6e64bf5c6571c51cbdfcbd0449f79992edab498800d9a35e1aff6e4bfa5e2958 model.layers.0.self_attn.q_proj.weight F32 64x64 16384 f8e1f5c956cadef6516731b7feeb50696c9aeca84c6758a2dbf40133610e3811 model.layers.0.self_attn.v_proj.weight F32 64x64 16384 c8fc3f824739c12326e4d2c214c00a29229143dd41cc7dc4ed937d37b215c478 model.layers.1.input_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.1.mlp.experts.0.down_proj.weight F32 64x32 8192 05590e29f6afa48d46a108e21592aa09435d4e6ea2c531d8d992a867a5721e07 model.layers.1.mlp.experts.0.gate_proj.weight F32 32x64 8192 256415725de5a5cc0d9a29eac1af49b40865af40480b18e98db6fbe90c8c5a23 model.layers.1.mlp.experts.0.up_proj.weight F32 32x64 8192 51256ebf6919fefc524dc713ecdde227cb0513bb9e4eb44b59e53e414dfa2d2e model.layers.1.mlp.experts.1.down_proj.weight F32 64x32 8192 2a8d8f8cb644047cc27085146a4443d098ae2f4b01f233e5c3d5a0289542cc78 model.layers.1.mlp.experts.1.gate_proj.weight F32 32x64 8192 bcbfe3bad5d9c47d69cb8e8f36967887298f1fced333d8ab676036dc6358b92e model.layers.1.mlp.experts.1.up_proj.weight F32 32x64 8192 de39df2b6a4d9783b07a487aeff178951b5aacab28698acc7d35311d3001eaf3 model.layers.1.mlp.experts.2.down_proj.weight F32 64x32 8192 54f49cd5fd25fc1c2710d72cc47b46608f1b60f880ea14f70e32e753b2261334 model.layers.1.mlp.experts.2.gate_proj.weight F32 32x64 8192 ccaa6f187bb865a0bacbb1da841227e5dbe61a7f001ccf0d192c5cc22c22bac0 model.layers.1.mlp.experts.2.up_proj.weight F32 32x64 8192 321fd25b697c0e441f0e2cbb718a65ea1faacefd47857d91ee6230aeb5c645e8 model.layers.1.mlp.experts.3.down_proj.weight F32 64x32 8192 585df86081ff1739232e8868844baa780ba993f7637bcc5563a55b9a632a2f68 model.layers.1.mlp.experts.3.gate_proj.weight F32 32x64 8192 3a327f200a05d080aced4ba5e48c569e2769eaa5395c5fa659e79b51706cbd98 model.layers.1.mlp.experts.3.up_proj.weight F32 32x64 8192 d8ad31ff8591cfdf4980dea298917f9a510186c70043a36cc308fc42740050aa model.layers.1.mlp.experts.4.down_proj.weight F32 64x32 8192 0edf8b9bb6a636b573a03ac1162eb6ab84545190d2a55e1272aee292d85b9fe7 model.layers.1.mlp.experts.4.gate_proj.weight F32 32x64 8192 f1be4325cf99b902e3a6db9ac1f863d083d4f56b7a5e3dbde61c9729d3357c10 model.layers.1.mlp.experts.4.up_proj.weight F32 32x64 8192 d1f7735d9e252497246921d73040a5a1ed3c3c421a395f44f8110040ec9dcf21 model.layers.1.mlp.experts.5.down_proj.weight F32 64x32 8192 2b97eb6dadea00398db8f77e1d5b35bd7d8eced289b8468fdab32cbdd41b0934 model.layers.1.mlp.experts.5.gate_proj.weight F32 32x64 8192 4a7ea09c1997a827cde1ca31d80fdc3dc980ffb2c0ba2a1f71a92ecc1f9382fb model.layers.1.mlp.experts.5.up_proj.weight F32 32x64 8192 e368354d7e76e24f721a6ff9d1fbf7999214d563a16c3151f986d75a82e0e189 model.layers.1.mlp.experts.6.down_proj.weight F32 64x32 8192 7e9fa7da4b44829f506de51f47baeb669c51789ad5a988434f75d6e3e2afdb29 model.layers.1.mlp.experts.6.gate_proj.weight F32 32x64 8192 ca11a0f51d3911c7f1fbf86f86b5af1c999172b82b53daf86ce7facf9072f931 model.layers.1.mlp.experts.6.up_proj.weight F32 32x64 8192 07ab50f11b50b2a07e21f281aee80e19c5fe70de3101c6c0b3d759879354a462 model.layers.1.mlp.experts.7.down_proj.weight F32 64x32 8192 aba8ca92203bf1efd86b22dea3a107e1bc03e7f3463a65f763c4dd185ead4d0b model.layers.1.mlp.experts.7.gate_proj.weight F32 32x64 8192 bd39e058ffa4d890f6c81ba28b901e1f3fc68af1e08468621bf2e4385c5b831d model.layers.1.mlp.experts.7.up_proj.weight F32 32x64 8192 c4fc2b443e0ceb731c2cf0096cffac57ee11b032763d902076495704824e1951 model.layers.1.mlp.gate.weight F32 8x64 2048 45f345f1dcca82e49b845028fc5aa849cf91abf3112c039cb44e2a19d76c9798 model.layers.1.mlp.shared_experts.down_proj.weight F32 64x32 8192 663424a943380867d892893020c7ae41dcc24c715d41208634ffaf1c0df9117f model.layers.1.mlp.shared_experts.gate_proj.weight F32 32x64 8192 96b0f35baf832a54455b625c742f308ec12f06c64c4ba094ef6b0f56cc1b0252 model.layers.1.mlp.shared_experts.up_proj.weight F32 32x64 8192 68fc26c0b149dcf8a73437017a40565197d37e30362eff4caa66899b14600236 model.layers.1.post_attention_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.1.self_attn.k_proj.weight F32 64x64 16384 029f6cab5217bcedf69e3c4689183237038a367b2ee9a21b3a89450a2a05447c model.layers.1.self_attn.o_proj.weight F32 64x64 16384 d865f12f38d6605395c25eb7e3dc73f50bfbc5e543715938bfa557351f03e6ed model.layers.1.self_attn.q_proj.weight F32 64x64 16384 0c680f5e2613e184e95ba63cdd17101faabd7c81bacdb6cbaf84a69fb318335d model.layers.1.self_attn.v_proj.weight F32 64x64 16384 a0f913925bc23368b46c4462b231a1d2161eca7268226c8f1176f29d56341ad7 model.layers.2.input_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.2.mlp.experts.0.down_proj.weight F32 64x32 8192 2ac52a671f611e180a47a5793ce5e95f345abfb0e9c64639b2adbc5aa8557123 model.layers.2.mlp.experts.0.gate_proj.weight F32 32x64 8192 052749089632d5b772a4f42b83734ae5c8ee899245369ff7980b47a14e22d4fc model.layers.2.mlp.experts.0.up_proj.weight F32 32x64 8192 1afbe6f1f72934beec98af1304dded6d4339b033838626c2159053b32d19dd46 model.layers.2.mlp.experts.1.down_proj.weight F32 64x32 8192 b15ea1250ea8a23ea862c038b2ab6f9bc1d4dc2820c22a7a97163bfed33eae95 model.layers.2.mlp.experts.1.gate_proj.weight F32 32x64 8192 589863f7af7068e8a9c0e30d5f4944f0a623e74587673a898de02ccdce5f7256 model.layers.2.mlp.experts.1.up_proj.weight F32 32x64 8192 10bd20cc0163ca01f07031510720c6c2b066820c08cd5daf8eae44a8bcb1f12c model.layers.2.mlp.experts.2.down_proj.weight F32 64x32 8192 1e198036f6b35484fd8a94ab00bd29e4872bfa193146ce3d7eb851a9c80dda9e model.layers.2.mlp.experts.2.gate_proj.weight F32 32x64 8192 6491d2d2087ce7c59fa6fe675bec088e9231d30631cb690c9b0973ecf9964a1b model.layers.2.mlp.experts.2.up_proj.weight F32 32x64 8192 c3e20a70781c1981a9f5cf88cbc8083e90d4a813dccab4661b323217ed779aa8 model.layers.2.mlp.experts.3.down_proj.weight F32 64x32 8192 410aa80b7339adb92f234acef595f8196633d82d9103580efda420acbc5a5a01 model.layers.2.mlp.experts.3.gate_proj.weight F32 32x64 8192 3626a46061b25a7583f88c0dc5e0ce4587daae7795508d32af3b752ec761fe4b model.layers.2.mlp.experts.3.up_proj.weight F32 32x64 8192 fec9367a9c6c5f9f90fc68a122fbe84bc69cee5bf659db491d9293623577de17 model.layers.2.mlp.experts.4.down_proj.weight F32 64x32 8192 bef985038407d4e5c991b8190918978ad021f9591bfd14145515a61599aaa0c2 model.layers.2.mlp.experts.4.gate_proj.weight F32 32x64 8192 5562932abb3b6129af32fff550a8f079be406b5162da875376122022a50bd42a model.layers.2.mlp.experts.4.up_proj.weight F32 32x64 8192 8c387d756821d0a568f0f8df8a2cd2cdd98a4c1f6a8b53c8bd5c68555cf5f1a2 model.layers.2.mlp.experts.5.down_proj.weight F32 64x32 8192 f87065596894fffacea1c3cf86273ea17f45957b2085774a8500d36ea887444e model.layers.2.mlp.experts.5.gate_proj.weight F32 32x64 8192 fd4954d5278c22dfa5f2c50bce03d37fd19cf233e49f85e1243799f0fa1b0771 model.layers.2.mlp.experts.5.up_proj.weight F32 32x64 8192 01fad104cc66a5d74a6a07508e54ac93df6d381a52dfacddcff3a66aea74e031 model.layers.2.mlp.experts.6.down_proj.weight F32 64x32 8192 61bb0b6a1d04132de53bd533763b8d45763d15230c8a913240a17718f4c05e12 model.layers.2.mlp.experts.6.gate_proj.weight F32 32x64 8192 e8dfe7debe831f986d8b3efeb1951e1994fd48077cc89043d162a27b594d2b79 model.layers.2.mlp.experts.6.up_proj.weight F32 32x64 8192 74a41ec6827e966e10c77946e869ccb1c4820e659d2a5bd12a5a39ae86b6b134 model.layers.2.mlp.experts.7.down_proj.weight F32 64x32 8192 1bd74097de29d52150cd36a9ba1fcd2ca911cb13c577c67ad053a33dcb857f75 model.layers.2.mlp.experts.7.gate_proj.weight F32 32x64 8192 45f0310beedfcb078cbb6341ddce13736cb6fee670e96fd06591f9cebeeeceaf model.layers.2.mlp.experts.7.up_proj.weight F32 32x64 8192 d8e2f27934e660a9919248e343fd33f302547d44bcbb31a014ee5d26b063e8c2 model.layers.2.mlp.gate.weight F32 8x64 2048 27eedd751bd2d2f9097427d03b5da89c1c4dfcfc66d009732e0b4b12fe3ff357 model.layers.2.mlp.shared_experts.down_proj.weight F32 64x32 8192 8d43c807f8f715e140f9e3387b3120b794567e49876eecae1733da30b9c75d23 model.layers.2.mlp.shared_experts.gate_proj.weight F32 32x64 8192 598b31d89163ae5d66bec8a1414dddf4e2c40fe5411cdd047657673ae8f087de model.layers.2.mlp.shared_experts.up_proj.weight F32 32x64 8192 fd091759466cf0c1d75b649950a50258aee9deef7b557d0b0c912fe350076b53 model.layers.2.post_attention_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.2.self_attn.k_proj.weight F32 64x64 16384 5ac524806435b0d04a800dbbe5d1eeb4ce89f5371d7a121f0fb294ee38aed5b6 model.layers.2.self_attn.o_proj.weight F32 64x64 16384 d2599ea288eb2fdb2325b49ec1831ccbed72a53317bcee507d833687c2015c7e model.layers.2.self_attn.q_proj.weight F32 64x64 16384 6b66e3aecb498ca633da02a2a17f11a7867e623a971747b327e182967329ddd2 model.layers.2.self_attn.v_proj.weight F32 64x64 16384 2b742d57d70e03bd307cafe90986b78d59ef1958a9506ae9b1e7951d3830fbf8 model.layers.3.input_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.3.mlp.experts.0.down_proj.weight F32 64x32 8192 654cdabf827682370914d3779a40c58ad290f6d4ed79d431e5b83c297f87d803 model.layers.3.mlp.experts.0.gate_proj.weight F32 32x64 8192 4566fba70c953931e1fa27d4582f13a9eb4c63d956507d73a089228834764683 model.layers.3.mlp.experts.0.up_proj.weight F32 32x64 8192 35d96d72b72ac4f7235cf348ae892cc3d1647572a31ad2b8d2cabb9b3cc49120 model.layers.3.mlp.experts.1.down_proj.weight F32 64x32 8192 4a1cde4b6534f0cc3d9bee02b8e40d8e42607de0f53e494f090c75d47bc5f136 model.layers.3.mlp.experts.1.gate_proj.weight F32 32x64 8192 8081976fbaecedd1ee7d6de0321285f656dcff39af7ae3d3cc24c639eec26ec7 model.layers.3.mlp.experts.1.up_proj.weight F32 32x64 8192 628fa3db25b7b353d24cbe56294663ecb73ada68d419b810b2163db70ee33fd9 model.layers.3.mlp.experts.2.down_proj.weight F32 64x32 8192 788bceca28b87368305c45e8c3024d6ac50c73411c11f31ef9eaf8c45da54b9d model.layers.3.mlp.experts.2.gate_proj.weight F32 32x64 8192 9420e9cb56571c08cefe677e98bf27465aa461e5a6a4ac10e91249afc7979adf model.layers.3.mlp.experts.2.up_proj.weight F32 32x64 8192 78aef1d5b08bf936ae868ecc143fddabc1ddec40c12037d9bcbaf9e95f8c0478 model.layers.3.mlp.experts.3.down_proj.weight F32 64x32 8192 da6c5bf5e5c7c4a81f793d7c403e7998ddf04972f8a7fe37c17f182fbc1e8fbd model.layers.3.mlp.experts.3.gate_proj.weight F32 32x64 8192 abb81b4cdf60b733008f6f2c10f5120d1069e5851d0b99e82faca572429acd1e model.layers.3.mlp.experts.3.up_proj.weight F32 32x64 8192 13d04ef08e3ec4bef343a4c146fa6d9288a1456aa1b2a08b8dc14f70ba677a61 model.layers.3.mlp.experts.4.down_proj.weight F32 64x32 8192 44b099e2d95164a3b92716185a9abfaf3c63d0e1eb46b5363bb42b8d0b9194a5 model.layers.3.mlp.experts.4.gate_proj.weight F32 32x64 8192 0539659f30e3dafc93a83a68842f8f9c83ec97d3b9a9ad3f289c28e1a244e6ff model.layers.3.mlp.experts.4.up_proj.weight F32 32x64 8192 a5932b71cecceb4405b1ef88b52b0e6bf426a4d6359e33bbade048e24df62ef9 model.layers.3.mlp.experts.5.down_proj.weight F32 64x32 8192 b60ee9d15e62e3fd893f39595d932f01cac819410510a1fb924680f6037b6313 model.layers.3.mlp.experts.5.gate_proj.weight F32 32x64 8192 76642070769d52bdebde246c3d4dafc888ee212eecd8a806547546593b5f340e model.layers.3.mlp.experts.5.up_proj.weight F32 32x64 8192 365930a51bf20916ba47bd7041e5b7dfc1c5e8ec10bdc0a0c6df914f81e08d43 model.layers.3.mlp.experts.6.down_proj.weight F32 64x32 8192 a82079e12cea314bfb38cf4c608c7bf2de9da7a443cc98cf9660df9dac413e02 model.layers.3.mlp.experts.6.gate_proj.weight F32 32x64 8192 03560ba6680cfde7a21d2a4c0de1a649b733e990eed356e2dca778d8f3aa4639 model.layers.3.mlp.experts.6.up_proj.weight F32 32x64 8192 79d3de521d17a6d379fcd6801fbadf819918a8768f067168ae2a595feca3dcd7 model.layers.3.mlp.experts.7.down_proj.weight F32 64x32 8192 bebdb5648ca35964bb86c0bbfe7209148c982abe522b1f0a59900f141de2ed5d model.layers.3.mlp.experts.7.gate_proj.weight F32 32x64 8192 4b8d349b5d9ed5c07a22683394ce182c6fc4dbb55b76c0bf08056055c2756a26 model.layers.3.mlp.experts.7.up_proj.weight F32 32x64 8192 7b369e548ff02420c7f5c4d8d46cd94211895e7b7b958730bca746bc8c345ea3 model.layers.3.mlp.gate.weight F32 8x64 2048 5e25406e819d5b5fbb4a44ff8ea56dd0965f1472ff9d95475b1a569022d322b3 model.layers.3.mlp.shared_experts.down_proj.weight F32 64x32 8192 16a05800c97aa10299481eb134090a91235f283555bc53d1a617e2424e2c4cc8 model.layers.3.mlp.shared_experts.gate_proj.weight F32 32x64 8192 da24440b5cc3e961320c43b574ab63997ad183857e4b23e0122d543a6d05de99 model.layers.3.mlp.shared_experts.up_proj.weight F32 32x64 8192 c7e869f338eb3bce124e1bf358500cbf229903c05848f8c9b39b2759f3dc4180 model.layers.3.post_attention_layernorm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0 model.layers.3.self_attn.k_proj.weight F32 64x64 16384 ee7a9a25e9783a39f671463aa4b06be02c3553b696e742cf3ecffcee953934a0 model.layers.3.self_attn.o_proj.weight F32 64x64 16384 af9721b7952417c2277d865939d302b1e6310ad97661600888c53e2c0886065d model.layers.3.self_attn.q_proj.weight F32 64x64 16384 58c3166923b98f7c9892d96ac3786b09cf24e8d4283e759278e4155e15ba7c9c model.layers.3.self_attn.v_proj.weight F32 64x64 16384 fea9682bf65244d775bafbca727169591cbace2fc65208beb2e083305cd65928 model.norm.weight F32 64 256 2f20cd03c9cd392a406c56232b0ff93a15f6d6d7da79086bfa14f55d4a4031b0