| { |
| "adapter": { |
| "config": { |
| "alora_invocation_tokens": null, |
| "alpha_pattern": {}, |
| "arrow_config": null, |
| "auto_mapping": null, |
| "base_model_name_or_path": "", |
| "bias": "none", |
| "corda_config": null, |
| "ensure_weight_tying": false, |
| "eva_config": null, |
| "exclude_modules": null, |
| "fan_in_fan_out": false, |
| "inference_mode": true, |
| "init_lora_weights": true, |
| "layer_replication": null, |
| "layers_pattern": null, |
| "layers_to_transform": null, |
| "loftq_config": {}, |
| "lora_alpha": 64, |
| "lora_bias": false, |
| "lora_dropout": 0.0, |
| "lora_ga_config": null, |
| "megatron_config": null, |
| "megatron_core": "megatron.core", |
| "modules_to_save": null, |
| "peft_type": "LORA", |
| "peft_version": "0.19.1", |
| "qalora_group_size": 16, |
| "r": 32, |
| "rank_pattern": {}, |
| "revision": null, |
| "target_modules": [ |
| "in_proj_qkv", |
| "in_proj_z", |
| "k_proj", |
| "o_proj", |
| "v_proj", |
| "q_proj", |
| "out_proj", |
| "in_proj_a", |
| "in_proj_b" |
| ], |
| "target_parameters": null, |
| "task_type": "CAUSAL_LM", |
| "trainable_token_indices": null, |
| "use_bdlora": null, |
| "use_dora": false, |
| "use_qalora": false, |
| "use_rslora": false |
| }, |
| "config_sha256": "294f6a66ddde05c2b38738d0577876060ded51e50712a689470e7c478fd808c3", |
| "local_dir": "/home/nathanbreslow/antidoomKLSFT/checkpoints/qwen36_35b_antidoom_kl_lora_r32_v2", |
| "model_sha256": "9347d68d937eecbbdb22be7232f8905692436346817697b02ebc94b1a1a53faf", |
| "training_args_sha256": "244a82e544375e45a0cfbd27e61e59784802255f85a8a8a3a813abfaeebae476" |
| }, |
| "base": { |
| "index_sha256": "41b9356101ebf8e7519e150dc811f80c4226e727301fbb032b890f006ed0be83", |
| "local_snapshot": "/home/nathanbreslow/.cache/huggingface/hub/models--Qwen--Qwen3.6-35B-A3B/snapshots/995ad96eacd98c81ed38be0c5b274b04031597b0", |
| "repo": "Qwen/Qwen3.6-35B-A3B", |
| "revision": "995ad96eacd98c81ed38be0c5b274b04031597b0", |
| "shards": 26 |
| }, |
| "created_at": "2026-07-09T22:37:29.967862+00:00", |
| "environment": { |
| "platform": "Linux-6.17.0-1021-nvidia-aarch64-with-glibc2.39", |
| "python": "3.12.13 (main, Jun 23 2026, 15:19:07) [Clang 22.1.3 ]", |
| "safetensors": "0.8.0", |
| "script_sha256": "e808446268e442a75bbe0ca52502f219e305dd1ac22d90b1cd50929ee92a6fcb", |
| "torch": "2.11.0+cu130" |
| }, |
| "merge": { |
| "formula": "BF16(W_FP32 + B_FP32 @ (A_FP32 * (lora_alpha / r)))", |
| "layer_counts": { |
| "0": 5, |
| "1": 5, |
| "10": 5, |
| "11": 4, |
| "12": 5, |
| "13": 5, |
| "14": 5, |
| "15": 4, |
| "16": 5, |
| "17": 5, |
| "18": 5, |
| "19": 4, |
| "2": 5, |
| "20": 5, |
| "21": 5, |
| "22": 5, |
| "23": 4, |
| "24": 5, |
| "25": 5, |
| "26": 5, |
| "27": 4, |
| "28": 5, |
| "29": 5, |
| "3": 4, |
| "30": 5, |
| "31": 4, |
| "32": 5, |
| "33": 5, |
| "34": 5, |
| "35": 4, |
| "36": 5, |
| "37": 5, |
| "38": 5, |
| "39": 4, |
| "4": 5, |
| "5": 5, |
| "6": 5, |
| "7": 4, |
| "8": 5, |
| "9": 5 |
| }, |
| "module_counts": { |
| "in_proj_a": 30, |
| "in_proj_b": 30, |
| "in_proj_qkv": 30, |
| "in_proj_z": 30, |
| "k_proj": 10, |
| "o_proj": 10, |
| "out_proj": 30, |
| "q_proj": 10, |
| "v_proj": 10 |
| }, |
| "strategy": "one target tensor at a time, row-tiled, in-place in a copied shard", |
| "target_count": 190, |
| "threads": 2, |
| "tile_rows": 32 |
| }, |
| "merge_seconds": 317.917, |
| "schema_version": 1, |
| "shards": { |
| "model-00001-of-00026.safetensors": { |
| "sha256": "2f29ad3f6b31540418066e66ea9a017c6f112d5ab185e57ba2f41c9fb705978e", |
| "size": 3996199712 |
| }, |
| "model-00002-of-00026.safetensors": { |
| "sha256": "02edba7a7845c762b9da61d736b8404d680c33c96687f957e767b44f7cff954c", |
| "size": 1284907696 |
| }, |
| "model-00003-of-00026.safetensors": { |
| "sha256": "39b9acfcfeb1d9062e449c61fe9a0e6975c40e28dae4f9b476358f6e5ccfeb74", |
| "size": 3357898360 |
| }, |
| "model-00004-of-00026.safetensors": { |
| "sha256": "972bae2f88c1369355ef84a863ddfa8a3c66b2029d54fdd4f3299f6b87475184", |
| "size": 3370808712 |
| }, |
| "model-00005-of-00026.safetensors": { |
| "sha256": "00609f3287a54c589843e5e41d6f1d83f2f4f83af49fab6192c3c207e74b7062", |
| "size": 3357898360 |
| }, |
| "model-00006-of-00026.safetensors": { |
| "sha256": "6175f8ffe687723d9b140fb9ca2eeb6c5705d46d0fd8b6afe291703db3f3e930", |
| "size": 3959424904 |
| }, |
| "model-00007-of-00026.safetensors": { |
| "sha256": "ab54c82bf2cb55dda4994c6651b95e6a41a1ac834f6ee883812743a3dc21fe99", |
| "size": 1096788232 |
| }, |
| "model-00008-of-00026.safetensors": { |
| "sha256": "0a3b4b8d0c96f9b42ed0ea1ca4d460a06a6032c7286a2a090060aac7605f8023", |
| "size": 3946842008 |
| }, |
| "model-00009-of-00026.safetensors": { |
| "sha256": "d940216c8c6c11cad0b32390a372c6b73fe420b86f04ae2bf4be18b7ce7818ec", |
| "size": 1096460848 |
| }, |
| "model-00010-of-00026.safetensors": { |
| "sha256": "001f7913d74dab95f9687006c65efbb958d652be7f0ecf3f7bda757cbd90f1f9", |
| "size": 3946841992 |
| }, |
| "model-00011-of-00026.safetensors": { |
| "sha256": "5a089da7bc158b113ed15a7f7d515f9311da156532eeab17ff6d7e4cef9679b4", |
| "size": 1096460752 |
| }, |
| "model-00012-of-00026.safetensors": { |
| "sha256": "d51714d98723f308ec5bb3137514951fa202abfc5e01b66c75181dd39804688a", |
| "size": 3409971080 |
| }, |
| "model-00013-of-00026.safetensors": { |
| "sha256": "36a6e8f572fca3310255fb001f6b116ae90781e2f4798eee6dc5a79ee92fcf8f", |
| "size": 1633331664 |
| }, |
| "model-00014-of-00026.safetensors": { |
| "sha256": "c03d665943d7dbbcc5016e9674180fb642f67c2cec4c768dff85e2f00ab0cc68", |
| "size": 3422553872 |
| }, |
| "model-00015-of-00026.safetensors": { |
| "sha256": "a54d60d24c7234952847c71c9c8eaa780136cb691d0e1254aa93b3fe11f922d1", |
| "size": 1633659224 |
| }, |
| "model-00016-of-00026.safetensors": { |
| "sha256": "856939b826bcae35bd6a88a0afbc04d838306712b3731b947927e59afee562d9", |
| "size": 3946842136 |
| }, |
| "model-00017-of-00026.safetensors": { |
| "sha256": "35058ac91b5652a294842a67d970e738c8ccb4b409e875565eb56199d3aa997a", |
| "size": 1096460608 |
| }, |
| "model-00018-of-00026.safetensors": { |
| "sha256": "6f02e3886d3ec19bf5e8ee4401f0b038ee4908ec277c9562262a019a55bc1984", |
| "size": 3946841992 |
| }, |
| "model-00019-of-00026.safetensors": { |
| "sha256": "e284dc89e707c65639714385c13b0a0148db71b685ae7ace9dcace0bc07dd5b6", |
| "size": 1096460808 |
| }, |
| "model-00020-of-00026.safetensors": { |
| "sha256": "365eb7ffa4efb30a9605eec3212462df30c1f92f0bc31b23ad88447601ff3026", |
| "size": 3409971072 |
| }, |
| "model-00021-of-00026.safetensors": { |
| "sha256": "d74daf37694a5d997cafe1d05fddc64aad7a868a210675c78e5e9d3f87150ab9", |
| "size": 1633331744 |
| }, |
| "model-00022-of-00026.safetensors": { |
| "sha256": "c9059e5816bb428e302a5fc575ad55fd3ea1c77ececde84f599d496834f8218b", |
| "size": 3370808752 |
| }, |
| "model-00023-of-00026.safetensors": { |
| "sha256": "c4261ba26ddd9b875d04bed2e35e5d93ba4cdda923d4b51f1bd476c310ad0acb", |
| "size": 3357898392 |
| }, |
| "model-00024-of-00026.safetensors": { |
| "sha256": "fd381409ec88968f177fc2968c1b311ec1bad48a8efaace6104918c439304758", |
| "size": 3370808752 |
| }, |
| "model-00025-of-00026.safetensors": { |
| "sha256": "4dc3961a2e10e6b01da2f19bf546b60e5fe8a520d219231b8ab38b618865fb13", |
| "size": 3832888256 |
| }, |
| "model-00026-of-00026.safetensors": { |
| "sha256": "a30d2c6c2a09f92bce458c4d42fbf4bc3410062d25f1a2d020ac42035c107b1a", |
| "size": 2231416848 |
| } |
| }, |
| "target_stats": { |
| "model.language_model.layers.0.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 30755, |
| "changed_fraction": 0.4692840576171875, |
| "elements": 65536, |
| "formula_nonzero_elements": 63488, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.0.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.25063905038321e-05, |
| "scaling": 2.0, |
| "shard": "model-00002-of-00026.safetensors" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 48709, |
| "changed_fraction": 0.7432403564453125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000579833984375, |
| "module": "base_model.model.model.layers.0.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.360447674059165e-05, |
| "scaling": 2.0, |
| "shard": "model-00002-of-00026.safetensors" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10751965, |
| "changed_fraction": 0.6408670544624329, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16547840, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.0.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.51468740055135e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5165585, |
| "changed_fraction": 0.6157857179641724, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.0.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.136164236255221e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.0.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4227429, |
| "changed_fraction": 0.5039488077163696, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00115966796875, |
| "module": "base_model.model.model.layers.0.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 5.702941032752764e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 33111, |
| "changed_fraction": 0.5052337646484375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000591278076171875, |
| "module": "base_model.model.model.layers.1.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 6.47281010775474e-05, |
| "scaling": 2.0, |
| "shard": "model-00002-of-00026.safetensors" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42255, |
| "changed_fraction": 0.6447601318359375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0003662109375, |
| "module": "base_model.model.model.layers.1.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.433693698526348e-05, |
| "scaling": 2.0, |
| "shard": "model-00002-of-00026.safetensors" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 9998779, |
| "changed_fraction": 0.5959736704826355, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16367615, |
| "max_abs_effective_bf16_delta": 0.0010216832160949707, |
| "module": "base_model.model.model.layers.1.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 6.986407024210961e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5231233, |
| "changed_fraction": 0.623611569404602, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.00091552734375, |
| "module": "base_model.model.model.layers.1.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.359320666631618e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.1.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4777187, |
| "changed_fraction": 0.569485068321228, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.1.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 5.66848069458663e-05, |
| "scaling": 2.0, |
| "shard": "model-00001-of-00026.safetensors" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 41894, |
| "changed_fraction": 0.639251708984375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.10.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.030118138951974e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46368, |
| "changed_fraction": 0.70751953125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000560760498046875, |
| "module": "base_model.model.model.layers.10.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.305791432245633e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10747736, |
| "changed_fraction": 0.6406149864196777, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16769023, |
| "max_abs_effective_bf16_delta": 0.0008058547973632812, |
| "module": "base_model.model.model.layers.10.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.884481753437252e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5717960, |
| "changed_fraction": 0.6816339492797852, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.10.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 8.197803025550057e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.10.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5380400, |
| "changed_fraction": 0.6413936614990234, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00091552734375, |
| "module": "base_model.model.model.layers.10.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.199138831694352e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.11.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 694448, |
| "changed_fraction": 0.6622772216796875, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.000579833984375, |
| "module": "base_model.model.model.layers.11.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.355739501388593e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.11.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5479007, |
| "changed_fraction": 0.6531485319137573, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.0009410381317138672, |
| "module": "base_model.model.model.layers.11.self_attn.o_proj", |
| "rms_effective_bf16_delta": 8.168481778209755e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.11.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10769572, |
| "changed_fraction": 0.6419165134429932, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.11.self_attn.q_proj", |
| "rms_effective_bf16_delta": 7.578644977082595e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.11.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 758420, |
| "changed_fraction": 0.7232856750488281, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.11.self_attn.v_proj", |
| "rms_effective_bf16_delta": 8.498003892171755e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42608, |
| "changed_fraction": 0.650146484375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.12.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.816997522647381e-05, |
| "scaling": 2.0, |
| "shard": "model-00009-of-00026.safetensors" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46557, |
| "changed_fraction": 0.7104034423828125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00054931640625, |
| "module": "base_model.model.model.layers.12.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.412433719402122e-05, |
| "scaling": 2.0, |
| "shard": "model-00009-of-00026.safetensors" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10868056, |
| "changed_fraction": 0.6477866172790527, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777213, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.12.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.18584051920274e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5882139, |
| "changed_fraction": 0.7012056112289429, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.12.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 8.230692583350476e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.12.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5516697, |
| "changed_fraction": 0.6576415300369263, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.12.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.846577771796752e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 41837, |
| "changed_fraction": 0.6383819580078125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000946044921875, |
| "module": "base_model.model.model.layers.13.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 9.347651483814202e-05, |
| "scaling": 2.0, |
| "shard": "model-00009-of-00026.safetensors" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46031, |
| "changed_fraction": 0.7023773193359375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000396728515625, |
| "module": "base_model.model.model.layers.13.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.12724770719925e-05, |
| "scaling": 2.0, |
| "shard": "model-00009-of-00026.safetensors" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10692640, |
| "changed_fraction": 0.6373310089111328, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.0012636184692382812, |
| "module": "base_model.model.model.layers.13.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.316534015371205e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5786985, |
| "changed_fraction": 0.6898623704910278, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0010528564453125, |
| "module": "base_model.model.model.layers.13.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 8.540713466959101e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.13.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5599044, |
| "changed_fraction": 0.6674580574035645, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388605, |
| "max_abs_effective_bf16_delta": 0.000885009765625, |
| "module": "base_model.model.model.layers.13.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.360459743314073e-05, |
| "scaling": 2.0, |
| "shard": "model-00008-of-00026.safetensors" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 36731, |
| "changed_fraction": 0.5604705810546875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.14.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 6.408820823201858e-05, |
| "scaling": 2.0, |
| "shard": "model-00011-of-00026.safetensors" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46324, |
| "changed_fraction": 0.70684814453125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000579833984375, |
| "module": "base_model.model.model.layers.14.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.931456048098573e-05, |
| "scaling": 2.0, |
| "shard": "model-00011-of-00026.safetensors" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10758927, |
| "changed_fraction": 0.6412820219993591, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.00091552734375, |
| "module": "base_model.model.model.layers.14.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.396889941829679e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6000591, |
| "changed_fraction": 0.715326189994812, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009002685546875, |
| "module": "base_model.model.model.layers.14.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 8.636677447278575e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.14.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5598240, |
| "changed_fraction": 0.6673622131347656, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.14.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.746053163095078e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.15.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 701780, |
| "changed_fraction": 0.6692695617675781, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.0008392333984375, |
| "module": "base_model.model.model.layers.15.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.595614539234427e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.15.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5682420, |
| "changed_fraction": 0.6773972511291504, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.15.self_attn.o_proj", |
| "rms_effective_bf16_delta": 9.326062588130585e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.15.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11393837, |
| "changed_fraction": 0.6791256070137024, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.15.self_attn.q_proj", |
| "rms_effective_bf16_delta": 8.019703041610845e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.15.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 764574, |
| "changed_fraction": 0.7291545867919922, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.15.self_attn.v_proj", |
| "rms_effective_bf16_delta": 9.223723003081227e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 40458, |
| "changed_fraction": 0.617340087890625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.16.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.926234263603284e-05, |
| "scaling": 2.0, |
| "shard": "model-00011-of-00026.safetensors" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47306, |
| "changed_fraction": 0.721832275390625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.16.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.947450366004857e-05, |
| "scaling": 2.0, |
| "shard": "model-00011-of-00026.safetensors" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10924269, |
| "changed_fraction": 0.6511371731758118, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.16.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.514577641788167e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5987956, |
| "changed_fraction": 0.7138199806213379, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.16.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.070390622614111e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.16.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5665606, |
| "changed_fraction": 0.6753928661346436, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.16.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.717200426859417e-05, |
| "scaling": 2.0, |
| "shard": "model-00010-of-00026.safetensors" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 40944, |
| "changed_fraction": 0.624755859375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.17.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.38633181213219e-05, |
| "scaling": 2.0, |
| "shard": "model-00013-of-00026.safetensors" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46734, |
| "changed_fraction": 0.713104248046875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.17.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.645919554666517e-05, |
| "scaling": 2.0, |
| "shard": "model-00013-of-00026.safetensors" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11070108, |
| "changed_fraction": 0.65982985496521, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.17.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.773317292698886e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6084979, |
| "changed_fraction": 0.7253860235214233, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.17.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.123145477755034e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.17.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5673400, |
| "changed_fraction": 0.6763219833374023, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.17.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 9.202713231300383e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42062, |
| "changed_fraction": 0.641815185546875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.18.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.428588345419565e-05, |
| "scaling": 2.0, |
| "shard": "model-00013-of-00026.safetensors" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47332, |
| "changed_fraction": 0.72222900390625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.18.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.459836919250263e-05, |
| "scaling": 2.0, |
| "shard": "model-00013-of-00026.safetensors" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11072675, |
| "changed_fraction": 0.6599828600883484, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.18.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 9.043640688973794e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5985994, |
| "changed_fraction": 0.7135860919952393, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0013427734375, |
| "module": "base_model.model.model.layers.18.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.505438980045448e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.18.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5617121, |
| "changed_fraction": 0.6696130037307739, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000946044921875, |
| "module": "base_model.model.model.layers.18.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.487318640617153e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.19.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 717909, |
| "changed_fraction": 0.6846513748168945, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.19.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.58642914683779e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.19.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5519486, |
| "changed_fraction": 0.6579740047454834, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.19.self_attn.o_proj", |
| "rms_effective_bf16_delta": 8.263213799470523e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.19.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11628234, |
| "changed_fraction": 0.6930967569351196, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.19.self_attn.q_proj", |
| "rms_effective_bf16_delta": 8.709391886705632e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.19.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 761626, |
| "changed_fraction": 0.7263431549072266, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.0009212493896484375, |
| "module": "base_model.model.model.layers.19.self_attn.v_proj", |
| "rms_effective_bf16_delta": 8.91430244772417e-05, |
| "scaling": 2.0, |
| "shard": "model-00012-of-00026.safetensors" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 39811, |
| "changed_fraction": 0.6074676513671875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.2.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.184019530288684e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 44642, |
| "changed_fraction": 0.681182861328125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000579833984375, |
| "module": "base_model.model.model.layers.2.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.96682201680039e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10180888, |
| "changed_fraction": 0.6068282127380371, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16547836, |
| "max_abs_effective_bf16_delta": 0.000995635986328125, |
| "module": "base_model.model.model.layers.2.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.100468354687935e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5378050, |
| "changed_fraction": 0.6411135196685791, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.2.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.283989786059287e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.2.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4510282, |
| "changed_fraction": 0.5376675128936768, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00093841552734375, |
| "module": "base_model.model.model.layers.2.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 6.235573763046415e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 41909, |
| "changed_fraction": 0.6394805908203125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.20.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.93851098858551e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46827, |
| "changed_fraction": 0.7145233154296875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.20.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.938891775580925e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10428442, |
| "changed_fraction": 0.6215835809707642, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.20.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.930851044544183e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5788078, |
| "changed_fraction": 0.6899926662445068, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00115203857421875, |
| "module": "base_model.model.model.layers.20.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.368472263909851e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.20.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5360389, |
| "changed_fraction": 0.6390081644058228, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.20.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.357498774166259e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 40619, |
| "changed_fraction": 0.6197967529296875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.21.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.828153390238863e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 48063, |
| "changed_fraction": 0.7333831787109375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.21.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 8.272542708067484e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10417391, |
| "changed_fraction": 0.6209248900413513, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16773120, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.21.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.82992226033504e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5771549, |
| "changed_fraction": 0.688022255897522, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.001621246337890625, |
| "module": "base_model.model.model.layers.21.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.694958700437348e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.21.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5266131, |
| "changed_fraction": 0.6277717351913452, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0007476806640625, |
| "module": "base_model.model.model.layers.21.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.394243018921675e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42877, |
| "changed_fraction": 0.6542510986328125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00054931640625, |
| "module": "base_model.model.model.layers.22.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.447058045108835e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45907, |
| "changed_fraction": 0.7004852294921875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000499725341796875, |
| "module": "base_model.model.model.layers.22.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.73146372057541e-05, |
| "scaling": 2.0, |
| "shard": "model-00015-of-00026.safetensors" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10554381, |
| "changed_fraction": 0.6290901303291321, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16769024, |
| "max_abs_effective_bf16_delta": 0.0010728836059570312, |
| "module": "base_model.model.model.layers.22.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.774579516589965e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5863367, |
| "changed_fraction": 0.6989678144454956, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00103759765625, |
| "module": "base_model.model.model.layers.22.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.434000093003106e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.22.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5293620, |
| "changed_fraction": 0.6310486793518066, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.22.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.249776368945531e-05, |
| "scaling": 2.0, |
| "shard": "model-00014-of-00026.safetensors" |
| }, |
| "model.language_model.layers.23.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 678159, |
| "changed_fraction": 0.6467428207397461, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.23.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.518396677676303e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.23.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5218370, |
| "changed_fraction": 0.6220781803131104, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.23.self_attn.o_proj", |
| "rms_effective_bf16_delta": 8.036047474386736e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.23.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10712835, |
| "changed_fraction": 0.6385347247123718, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.23.self_attn.q_proj", |
| "rms_effective_bf16_delta": 8.438226304057198e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.23.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 746834, |
| "changed_fraction": 0.7122364044189453, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.0007476806640625, |
| "module": "base_model.model.model.layers.23.self_attn.v_proj", |
| "rms_effective_bf16_delta": 8.178560910881905e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 40637, |
| "changed_fraction": 0.6200714111328125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.24.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 6.87900255946991e-05, |
| "scaling": 2.0, |
| "shard": "model-00017-of-00026.safetensors" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45379, |
| "changed_fraction": 0.6924285888671875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.24.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.982129294249513e-05, |
| "scaling": 2.0, |
| "shard": "model-00017-of-00026.safetensors" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10362982, |
| "changed_fraction": 0.6176818609237671, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.24.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.51102613002069e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5883216, |
| "changed_fraction": 0.7013339996337891, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.24.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.154575677870438e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.24.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5249409, |
| "changed_fraction": 0.625778317451477, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.24.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.722079186994334e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 43391, |
| "changed_fraction": 0.6620941162109375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0007476806640625, |
| "module": "base_model.model.model.layers.25.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 9.756493403597129e-05, |
| "scaling": 2.0, |
| "shard": "model-00017-of-00026.safetensors" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47230, |
| "changed_fraction": 0.720672607421875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.25.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.860814471744432e-05, |
| "scaling": 2.0, |
| "shard": "model-00017-of-00026.safetensors" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10332363, |
| "changed_fraction": 0.6158568263053894, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.001068115234375, |
| "module": "base_model.model.model.layers.25.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.739930282826853e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5786058, |
| "changed_fraction": 0.6897518634796143, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00103759765625, |
| "module": "base_model.model.model.layers.25.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.676194801627504e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.25.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5281483, |
| "changed_fraction": 0.6296018362045288, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00067138671875, |
| "module": "base_model.model.model.layers.25.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.813189765614977e-05, |
| "scaling": 2.0, |
| "shard": "model-00016-of-00026.safetensors" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 39782, |
| "changed_fraction": 0.607025146484375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.26.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.568452986791395e-05, |
| "scaling": 2.0, |
| "shard": "model-00019-of-00026.safetensors" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45840, |
| "changed_fraction": 0.699462890625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.26.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 8.063676498761255e-05, |
| "scaling": 2.0, |
| "shard": "model-00019-of-00026.safetensors" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10485047, |
| "changed_fraction": 0.6249575018882751, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.26.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 8.989223925345269e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6003196, |
| "changed_fraction": 0.7156367301940918, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388604, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.26.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 9.987351500103812e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.26.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5295000, |
| "changed_fraction": 0.6312131881713867, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.26.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.994238414767838e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.27.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 683967, |
| "changed_fraction": 0.6522817611694336, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.0004901885986328125, |
| "module": "base_model.model.model.layers.27.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.73319782163451e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.27.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5308128, |
| "changed_fraction": 0.6327781677246094, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.27.self_attn.o_proj", |
| "rms_effective_bf16_delta": 8.711702037703368e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.27.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11430191, |
| "changed_fraction": 0.6812924742698669, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.001087188720703125, |
| "module": "base_model.model.model.layers.27.self_attn.q_proj", |
| "rms_effective_bf16_delta": 9.126735428806502e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.27.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 738739, |
| "changed_fraction": 0.7045164108276367, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048575, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.27.self_attn.v_proj", |
| "rms_effective_bf16_delta": 8.949030938293286e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 43108, |
| "changed_fraction": 0.65777587890625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00054931640625, |
| "module": "base_model.model.model.layers.28.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.5556712585574e-05, |
| "scaling": 2.0, |
| "shard": "model-00019-of-00026.safetensors" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47219, |
| "changed_fraction": 0.7205047607421875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.28.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.959844718254463e-05, |
| "scaling": 2.0, |
| "shard": "model-00019-of-00026.safetensors" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10649242, |
| "changed_fraction": 0.6347442865371704, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.0014352798461914062, |
| "module": "base_model.model.model.layers.28.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 9.628502558009477e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5957038, |
| "changed_fraction": 0.7101342678070068, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.28.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010274432978587353, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.28.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5364574, |
| "changed_fraction": 0.6395070552825928, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.28.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.804179398761905e-05, |
| "scaling": 2.0, |
| "shard": "model-00018-of-00026.safetensors" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42884, |
| "changed_fraction": 0.65435791015625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.29.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.808322381583734e-05, |
| "scaling": 2.0, |
| "shard": "model-00021-of-00026.safetensors" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46574, |
| "changed_fraction": 0.710662841796875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.29.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 8.585155676340322e-05, |
| "scaling": 2.0, |
| "shard": "model-00021-of-00026.safetensors" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10667129, |
| "changed_fraction": 0.6358104348182678, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.00128173828125, |
| "module": "base_model.model.model.layers.29.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 9.849695659035123e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6084440, |
| "changed_fraction": 0.7253217697143555, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00115966796875, |
| "module": "base_model.model.model.layers.29.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00011035977036764194, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.29.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5370665, |
| "changed_fraction": 0.6402331590652466, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.001007080078125, |
| "module": "base_model.model.model.layers.29.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.975065824565774e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.3.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 662135, |
| "changed_fraction": 0.6314611434936523, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.000682830810546875, |
| "module": "base_model.model.model.layers.3.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.241444047012491e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.3.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5041438, |
| "changed_fraction": 0.6009862422943115, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0005035400390625, |
| "module": "base_model.model.model.layers.3.self_attn.o_proj", |
| "rms_effective_bf16_delta": 6.026982563617216e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.3.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10113617, |
| "changed_fraction": 0.6028185486793518, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.00092315673828125, |
| "module": "base_model.model.model.layers.3.self_attn.q_proj", |
| "rms_effective_bf16_delta": 7.350570200486596e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.3.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 721849, |
| "changed_fraction": 0.6884088516235352, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048575, |
| "max_abs_effective_bf16_delta": 0.000926971435546875, |
| "module": "base_model.model.model.layers.3.self_attn.v_proj", |
| "rms_effective_bf16_delta": 7.747421220958643e-05, |
| "scaling": 2.0, |
| "shard": "model-00003-of-00026.safetensors" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 43839, |
| "changed_fraction": 0.6689300537109375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000885009765625, |
| "module": "base_model.model.model.layers.30.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 9.872395338817168e-05, |
| "scaling": 2.0, |
| "shard": "model-00021-of-00026.safetensors" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46826, |
| "changed_fraction": 0.714508056640625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000518798828125, |
| "module": "base_model.model.model.layers.30.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 8.24930908855796e-05, |
| "scaling": 2.0, |
| "shard": "model-00021-of-00026.safetensors" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10720563, |
| "changed_fraction": 0.638995349407196, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.00112152099609375, |
| "module": "base_model.model.model.layers.30.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 9.983738297083213e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6054666, |
| "changed_fraction": 0.7217724323272705, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0014801025390625, |
| "module": "base_model.model.model.layers.30.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010544629155757812, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.30.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5355054, |
| "changed_fraction": 0.6383721828460693, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.30.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.681716448816516e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.31.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 696095, |
| "changed_fraction": 0.6638479232788086, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.0009212493896484375, |
| "module": "base_model.model.model.layers.31.self_attn.k_proj", |
| "rms_effective_bf16_delta": 8.437487298802524e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.31.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5254913, |
| "changed_fraction": 0.6264344453811646, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00128173828125, |
| "module": "base_model.model.model.layers.31.self_attn.o_proj", |
| "rms_effective_bf16_delta": 8.925954272837241e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.31.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11193374, |
| "changed_fraction": 0.6671770811080933, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.00146484375, |
| "module": "base_model.model.model.layers.31.self_attn.q_proj", |
| "rms_effective_bf16_delta": 9.08660571373039e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.31.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 744481, |
| "changed_fraction": 0.7099924087524414, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00115966796875, |
| "module": "base_model.model.model.layers.31.self_attn.v_proj", |
| "rms_effective_bf16_delta": 9.294866747722607e-05, |
| "scaling": 2.0, |
| "shard": "model-00020-of-00026.safetensors" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 42686, |
| "changed_fraction": 0.651336669921875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.32.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.927647995249258e-05, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 48385, |
| "changed_fraction": 0.7382965087890625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.000701904296875, |
| "module": "base_model.model.model.layers.32.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 9.253677487914579e-05, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10909815, |
| "changed_fraction": 0.6502756476402283, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0011749267578125, |
| "module": "base_model.model.model.layers.32.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 0.00010049900326370136, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5980567, |
| "changed_fraction": 0.7129391431808472, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388605, |
| "max_abs_effective_bf16_delta": 0.001491546630859375, |
| "module": "base_model.model.model.layers.32.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010056438913395777, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.32.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5391896, |
| "changed_fraction": 0.6427640914916992, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00093841552734375, |
| "module": "base_model.model.model.layers.32.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.832600221432643e-05, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 44844, |
| "changed_fraction": 0.68426513671875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.33.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 0.00010916238745063663, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 48673, |
| "changed_fraction": 0.7426910400390625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00055694580078125, |
| "module": "base_model.model.model.layers.33.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 9.246898753329683e-05, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10909430, |
| "changed_fraction": 0.6502526998519897, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.001220703125, |
| "module": "base_model.model.model.layers.33.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 0.00010028798313212791, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5999651, |
| "changed_fraction": 0.7152141332626343, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.001190185546875, |
| "module": "base_model.model.model.layers.33.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010424333896815218, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.33.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5184738, |
| "changed_fraction": 0.6180689334869385, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00115966796875, |
| "module": "base_model.model.model.layers.33.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.289390623967547e-05, |
| "scaling": 2.0, |
| "shard": "model-00022-of-00026.safetensors" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 40663, |
| "changed_fraction": 0.6204681396484375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0008544921875, |
| "module": "base_model.model.model.layers.34.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.94923818246955e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46075, |
| "changed_fraction": 0.7030487060546875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0003662109375, |
| "module": "base_model.model.model.layers.34.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.623493961533599e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10708613, |
| "changed_fraction": 0.6382830739021301, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.34.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 9.605744535491254e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5958854, |
| "changed_fraction": 0.710350751876831, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00115966796875, |
| "module": "base_model.model.model.layers.34.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010116970767495327, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.34.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5518636, |
| "changed_fraction": 0.6578726768493652, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.34.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 9.38996420599654e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.35.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 690388, |
| "changed_fraction": 0.6584053039550781, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00054931640625, |
| "module": "base_model.model.model.layers.35.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.802179036060055e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.35.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5132304, |
| "changed_fraction": 0.6118183135986328, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.35.self_attn.o_proj", |
| "rms_effective_bf16_delta": 9.380541725705772e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.35.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11146707, |
| "changed_fraction": 0.6643955111503601, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0013427734375, |
| "module": "base_model.model.model.layers.35.self_attn.q_proj", |
| "rms_effective_bf16_delta": 8.689808335234204e-05, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.35.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 729376, |
| "changed_fraction": 0.695587158203125, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.001323699951171875, |
| "module": "base_model.model.model.layers.35.self_attn.v_proj", |
| "rms_effective_bf16_delta": 0.00010167408347723381, |
| "scaling": 2.0, |
| "shard": "model-00023-of-00026.safetensors" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 44847, |
| "changed_fraction": 0.6843109130859375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.36.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 0.00011075183923671724, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 50187, |
| "changed_fraction": 0.7657928466796875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.36.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 9.08482572719799e-05, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10660430, |
| "changed_fraction": 0.6354111433029175, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.001995086669921875, |
| "module": "base_model.model.model.layers.36.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 0.00010201097014785468, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6022287, |
| "changed_fraction": 0.7179125547409058, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.36.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010200691580254258, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.36.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5259156, |
| "changed_fraction": 0.6269402503967285, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.36.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 8.514353110152134e-05, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 41306, |
| "changed_fraction": 0.630279541015625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.37.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 9.075526212392864e-05, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47667, |
| "changed_fraction": 0.7273406982421875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65535, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.37.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 8.455174747374792e-05, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10697563, |
| "changed_fraction": 0.6376244425773621, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777213, |
| "max_abs_effective_bf16_delta": 0.0012664794921875, |
| "module": "base_model.model.model.layers.37.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 0.00010047640710325602, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6133336, |
| "changed_fraction": 0.7311506271362305, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.00140380859375, |
| "module": "base_model.model.model.layers.37.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00010564545033233658, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.37.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5651545, |
| "changed_fraction": 0.67371666431427, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0010833740234375, |
| "module": "base_model.model.model.layers.37.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 0.00010830739000305004, |
| "scaling": 2.0, |
| "shard": "model-00024-of-00026.safetensors" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 41451, |
| "changed_fraction": 0.6324920654296875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.38.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 9.510995748775557e-05, |
| "scaling": 2.0, |
| "shard": "model-00025-of-00026.safetensors" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 46651, |
| "changed_fraction": 0.7118377685546875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00054931640625, |
| "module": "base_model.model.model.layers.38.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 9.034939394162375e-05, |
| "scaling": 2.0, |
| "shard": "model-00025-of-00026.safetensors" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10948974, |
| "changed_fraction": 0.6526097059249878, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.00146484375, |
| "module": "base_model.model.model.layers.38.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 0.00011139941384783658, |
| "scaling": 2.0, |
| "shard": "model-00025-of-00026.safetensors" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 6145820, |
| "changed_fraction": 0.7326388359069824, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00146484375, |
| "module": "base_model.model.model.layers.38.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 0.00011233124404451418, |
| "scaling": 2.0, |
| "shard": "model-00025-of-00026.safetensors" |
| }, |
| "model.language_model.layers.38.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5734733, |
| "changed_fraction": 0.6836334466934204, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.38.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 0.00011524470527460758, |
| "scaling": 2.0, |
| "shard": "model-00025-of-00026.safetensors" |
| }, |
| "model.language_model.layers.39.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 744442, |
| "changed_fraction": 0.7099552154541016, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.39.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.563793790538429e-05, |
| "scaling": 2.0, |
| "shard": "model-00026-of-00026.safetensors" |
| }, |
| "model.language_model.layers.39.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5507689, |
| "changed_fraction": 0.6565676927566528, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.39.self_attn.o_proj", |
| "rms_effective_bf16_delta": 0.00010270824831516126, |
| "scaling": 2.0, |
| "shard": "model-00026-of-00026.safetensors" |
| }, |
| "model.language_model.layers.39.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 11262917, |
| "changed_fraction": 0.6713221669197083, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.0010986328125, |
| "module": "base_model.model.model.layers.39.self_attn.q_proj", |
| "rms_effective_bf16_delta": 8.799169598088403e-05, |
| "scaling": 2.0, |
| "shard": "model-00026-of-00026.safetensors" |
| }, |
| "model.language_model.layers.39.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 830406, |
| "changed_fraction": 0.7919368743896484, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.001190185546875, |
| "module": "base_model.model.model.layers.39.self_attn.v_proj", |
| "rms_effective_bf16_delta": 0.00011691026157727887, |
| "scaling": 2.0, |
| "shard": "model-00026-of-00026.safetensors" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 39980, |
| "changed_fraction": 0.61004638671875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.4.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 8.024908435996157e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45411, |
| "changed_fraction": 0.6929168701171875, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.4.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.820581114168836e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10472535, |
| "changed_fraction": 0.6242117285728455, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777216, |
| "max_abs_effective_bf16_delta": 0.000762939453125, |
| "module": "base_model.model.model.layers.4.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.423169572809558e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5458783, |
| "changed_fraction": 0.6507376432418823, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.000732421875, |
| "module": "base_model.model.model.layers.4.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.415020020046006e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.4.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4842240, |
| "changed_fraction": 0.577239990234375, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.4.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 5.6341563341890654e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 36954, |
| "changed_fraction": 0.563873291015625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00067138671875, |
| "module": "base_model.model.model.layers.5.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.908453169709464e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45055, |
| "changed_fraction": 0.6874847412109375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00030517578125, |
| "module": "base_model.model.model.layers.5.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.656953782415742e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10373943, |
| "changed_fraction": 0.6183351874351501, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.5.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.285539636718919e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5398227, |
| "changed_fraction": 0.6435188055038452, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.5.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.219678309162771e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.5.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4970646, |
| "changed_fraction": 0.5925471782684326, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.5.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 6.0648198292888544e-05, |
| "scaling": 2.0, |
| "shard": "model-00004-of-00026.safetensors" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 38780, |
| "changed_fraction": 0.59173583984375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0006103515625, |
| "module": "base_model.model.model.layers.6.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.117397091117397e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 43242, |
| "changed_fraction": 0.659820556640625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0003739595413208008, |
| "module": "base_model.model.model.layers.6.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.940429126071299e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10351489, |
| "changed_fraction": 0.6169968247413635, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.001117706298828125, |
| "module": "base_model.model.model.layers.6.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.364370105770223e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5560208, |
| "changed_fraction": 0.6628284454345703, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.6.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.529367979499003e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.6.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 4966000, |
| "changed_fraction": 0.5919933319091797, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.6.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 5.961404818933803e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.7.self_attn.k_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 661506, |
| "changed_fraction": 0.6308612823486328, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.7.self_attn.k_proj", |
| "rms_effective_bf16_delta": 7.067536790705216e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.7.self_attn.o_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5113964, |
| "changed_fraction": 0.6096320152282715, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.7.self_attn.o_proj", |
| "rms_effective_bf16_delta": 7.080729971930253e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.7.self_attn.q_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10590889, |
| "changed_fraction": 0.631266176700592, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777215, |
| "max_abs_effective_bf16_delta": 0.0009765625, |
| "module": "base_model.model.model.layers.7.self_attn.q_proj", |
| "rms_effective_bf16_delta": 7.40017609793409e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.7.self_attn.v_proj.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 512, |
| 32 |
| ], |
| "base_shape": [ |
| 512, |
| 2048 |
| ], |
| "changed_elements": 719821, |
| "changed_fraction": 0.6864748001098633, |
| "elements": 1048576, |
| "formula_nonzero_elements": 1048576, |
| "max_abs_effective_bf16_delta": 0.000949859619140625, |
| "module": "base_model.model.model.layers.7.self_attn.v_proj", |
| "rms_effective_bf16_delta": 8.238527252719216e-05, |
| "scaling": 2.0, |
| "shard": "model-00005-of-00026.safetensors" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 36348, |
| "changed_fraction": 0.55462646484375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.0003662109375, |
| "module": "base_model.model.model.layers.8.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 6.411371916081861e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 47509, |
| "changed_fraction": 0.7249298095703125, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.8.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 6.999336439184671e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10457812, |
| "changed_fraction": 0.6233341693878174, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16777214, |
| "max_abs_effective_bf16_delta": 0.0010771006345748901, |
| "module": "base_model.model.model.layers.8.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.903955210642053e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5633351, |
| "changed_fraction": 0.6715477705001831, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388607, |
| "max_abs_effective_bf16_delta": 0.00103759765625, |
| "module": "base_model.model.model.layers.8.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 8.130170606636331e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.8.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5311705, |
| "changed_fraction": 0.6332045793533325, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00079345703125, |
| "module": "base_model.model.model.layers.8.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.404814731691594e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_a.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 39113, |
| "changed_fraction": 0.5968170166015625, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00048828125, |
| "module": "base_model.model.model.layers.9.linear_attn.in_proj_a", |
| "rms_effective_bf16_delta": 7.49421331488855e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_b.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 32, |
| 32 |
| ], |
| "base_shape": [ |
| 32, |
| 2048 |
| ], |
| "changed_elements": 45887, |
| "changed_fraction": 0.7001800537109375, |
| "elements": 65536, |
| "formula_nonzero_elements": 65536, |
| "max_abs_effective_bf16_delta": 0.00042724609375, |
| "module": "base_model.model.model.layers.9.linear_attn.in_proj_b", |
| "rms_effective_bf16_delta": 7.018184983915064e-05, |
| "scaling": 2.0, |
| "shard": "model-00007-of-00026.safetensors" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 8192, |
| 32 |
| ], |
| "base_shape": [ |
| 8192, |
| 2048 |
| ], |
| "changed_elements": 10470688, |
| "changed_fraction": 0.6241016387939453, |
| "elements": 16777216, |
| "formula_nonzero_elements": 16773118, |
| "max_abs_effective_bf16_delta": 0.000946044921875, |
| "module": "base_model.model.model.layers.9.linear_attn.in_proj_qkv", |
| "rms_effective_bf16_delta": 7.70431663340467e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_z.weight": { |
| "a_shape": [ |
| 32, |
| 2048 |
| ], |
| "b_shape": [ |
| 4096, |
| 32 |
| ], |
| "base_shape": [ |
| 4096, |
| 2048 |
| ], |
| "changed_elements": 5617997, |
| "changed_fraction": 0.6697174310684204, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388608, |
| "max_abs_effective_bf16_delta": 0.00091552734375, |
| "module": "base_model.model.model.layers.9.linear_attn.in_proj_z", |
| "rms_effective_bf16_delta": 7.911620981435632e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| }, |
| "model.language_model.layers.9.linear_attn.out_proj.weight": { |
| "a_shape": [ |
| 32, |
| 4096 |
| ], |
| "b_shape": [ |
| 2048, |
| 32 |
| ], |
| "base_shape": [ |
| 2048, |
| 4096 |
| ], |
| "changed_elements": 5346855, |
| "changed_fraction": 0.6373947858810425, |
| "elements": 8388608, |
| "formula_nonzero_elements": 8388606, |
| "max_abs_effective_bf16_delta": 0.0007781982421875, |
| "module": "base_model.model.model.layers.9.linear_attn.out_proj", |
| "rms_effective_bf16_delta": 7.257326413407083e-05, |
| "scaling": 2.0, |
| "shard": "model-00006-of-00026.safetensors" |
| } |
| } |
| } |
|
|
