diff --git "a/merge_manifest.json" "b/merge_manifest.json" new file mode 100644--- /dev/null +++ "b/merge_manifest.json" @@ -0,0 +1,4613 @@ +{ + "adapter": { + "config": { + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "in_proj_qkv", + "in_proj_z", + "k_proj", + "o_proj", + "v_proj", + "q_proj", + "out_proj", + "in_proj_a", + "in_proj_b" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false + }, + "config_sha256": "294f6a66ddde05c2b38738d0577876060ded51e50712a689470e7c478fd808c3", + "local_dir": "/home/nathanbreslow/antidoomKLSFT/checkpoints/qwen36_35b_antidoom_kl_lora_r32_v2", + "model_sha256": "9347d68d937eecbbdb22be7232f8905692436346817697b02ebc94b1a1a53faf", + "training_args_sha256": "244a82e544375e45a0cfbd27e61e59784802255f85a8a8a3a813abfaeebae476" + }, + "base": { + "index_sha256": "41b9356101ebf8e7519e150dc811f80c4226e727301fbb032b890f006ed0be83", + "local_snapshot": "/home/nathanbreslow/.cache/huggingface/hub/models--Qwen--Qwen3.6-35B-A3B/snapshots/995ad96eacd98c81ed38be0c5b274b04031597b0", + "repo": "Qwen/Qwen3.6-35B-A3B", + "revision": "995ad96eacd98c81ed38be0c5b274b04031597b0", + "shards": 26 + }, + "created_at": "2026-07-09T22:37:29.967862+00:00", + "environment": { + "platform": "Linux-6.17.0-1021-nvidia-aarch64-with-glibc2.39", + "python": "3.12.13 (main, Jun 23 2026, 15:19:07) [Clang 22.1.3 ]", + "safetensors": "0.8.0", + "script_sha256": "e808446268e442a75bbe0ca52502f219e305dd1ac22d90b1cd50929ee92a6fcb", + "torch": "2.11.0+cu130" + }, + "merge": { + "formula": "BF16(W_FP32 + B_FP32 @ (A_FP32 * (lora_alpha / r)))", + "layer_counts": { + "0": 5, + "1": 5, + "10": 5, + "11": 4, + "12": 5, + "13": 5, + "14": 5, + "15": 4, + "16": 5, + "17": 5, + "18": 5, + "19": 4, + "2": 5, + "20": 5, + "21": 5, + "22": 5, + "23": 4, + "24": 5, + "25": 5, + "26": 5, + "27": 4, + "28": 5, + "29": 5, + "3": 4, + "30": 5, + "31": 4, + "32": 5, + "33": 5, + "34": 5, + "35": 4, + "36": 5, + "37": 5, + "38": 5, + "39": 4, + "4": 5, + "5": 5, + "6": 5, + "7": 4, + "8": 5, + "9": 5 + }, + "module_counts": { + "in_proj_a": 30, + "in_proj_b": 30, + "in_proj_qkv": 30, + "in_proj_z": 30, + "k_proj": 10, + "o_proj": 10, + "out_proj": 30, + "q_proj": 10, + "v_proj": 10 + }, + "strategy": "one target tensor at a time, row-tiled, in-place in a copied shard", + "target_count": 190, + "threads": 2, + "tile_rows": 32 + }, + "merge_seconds": 317.917, + "schema_version": 1, + "shards": { + "model-00001-of-00026.safetensors": { + "sha256": "2f29ad3f6b31540418066e66ea9a017c6f112d5ab185e57ba2f41c9fb705978e", + "size": 3996199712 + }, + "model-00002-of-00026.safetensors": { + "sha256": "02edba7a7845c762b9da61d736b8404d680c33c96687f957e767b44f7cff954c", + "size": 1284907696 + }, + "model-00003-of-00026.safetensors": { + "sha256": "39b9acfcfeb1d9062e449c61fe9a0e6975c40e28dae4f9b476358f6e5ccfeb74", + "size": 3357898360 + }, + "model-00004-of-00026.safetensors": { + "sha256": "972bae2f88c1369355ef84a863ddfa8a3c66b2029d54fdd4f3299f6b87475184", + "size": 3370808712 + }, + "model-00005-of-00026.safetensors": { + "sha256": "00609f3287a54c589843e5e41d6f1d83f2f4f83af49fab6192c3c207e74b7062", + "size": 3357898360 + }, + "model-00006-of-00026.safetensors": { + "sha256": "6175f8ffe687723d9b140fb9ca2eeb6c5705d46d0fd8b6afe291703db3f3e930", + "size": 3959424904 + }, + "model-00007-of-00026.safetensors": { + "sha256": "ab54c82bf2cb55dda4994c6651b95e6a41a1ac834f6ee883812743a3dc21fe99", + "size": 1096788232 + }, + "model-00008-of-00026.safetensors": { + "sha256": "0a3b4b8d0c96f9b42ed0ea1ca4d460a06a6032c7286a2a090060aac7605f8023", + "size": 3946842008 + }, + "model-00009-of-00026.safetensors": { + "sha256": "d940216c8c6c11cad0b32390a372c6b73fe420b86f04ae2bf4be18b7ce7818ec", + "size": 1096460848 + }, + "model-00010-of-00026.safetensors": { + "sha256": "001f7913d74dab95f9687006c65efbb958d652be7f0ecf3f7bda757cbd90f1f9", + "size": 3946841992 + }, + "model-00011-of-00026.safetensors": { + "sha256": "5a089da7bc158b113ed15a7f7d515f9311da156532eeab17ff6d7e4cef9679b4", + "size": 1096460752 + }, + "model-00012-of-00026.safetensors": { + "sha256": "d51714d98723f308ec5bb3137514951fa202abfc5e01b66c75181dd39804688a", + "size": 3409971080 + }, + "model-00013-of-00026.safetensors": { + "sha256": "36a6e8f572fca3310255fb001f6b116ae90781e2f4798eee6dc5a79ee92fcf8f", + "size": 1633331664 + }, + "model-00014-of-00026.safetensors": { + "sha256": "c03d665943d7dbbcc5016e9674180fb642f67c2cec4c768dff85e2f00ab0cc68", + "size": 3422553872 + }, + "model-00015-of-00026.safetensors": { + "sha256": "a54d60d24c7234952847c71c9c8eaa780136cb691d0e1254aa93b3fe11f922d1", + "size": 1633659224 + }, + "model-00016-of-00026.safetensors": { + "sha256": "856939b826bcae35bd6a88a0afbc04d838306712b3731b947927e59afee562d9", + "size": 3946842136 + }, + "model-00017-of-00026.safetensors": { + "sha256": "35058ac91b5652a294842a67d970e738c8ccb4b409e875565eb56199d3aa997a", + "size": 1096460608 + }, + "model-00018-of-00026.safetensors": { + "sha256": "6f02e3886d3ec19bf5e8ee4401f0b038ee4908ec277c9562262a019a55bc1984", + "size": 3946841992 + }, + "model-00019-of-00026.safetensors": { + "sha256": "e284dc89e707c65639714385c13b0a0148db71b685ae7ace9dcace0bc07dd5b6", + "size": 1096460808 + }, + "model-00020-of-00026.safetensors": { + "sha256": "365eb7ffa4efb30a9605eec3212462df30c1f92f0bc31b23ad88447601ff3026", + "size": 3409971072 + }, + "model-00021-of-00026.safetensors": { + "sha256": "d74daf37694a5d997cafe1d05fddc64aad7a868a210675c78e5e9d3f87150ab9", + "size": 1633331744 + }, + "model-00022-of-00026.safetensors": { + "sha256": "c9059e5816bb428e302a5fc575ad55fd3ea1c77ececde84f599d496834f8218b", + "size": 3370808752 + }, + "model-00023-of-00026.safetensors": { + "sha256": "c4261ba26ddd9b875d04bed2e35e5d93ba4cdda923d4b51f1bd476c310ad0acb", + "size": 3357898392 + }, + "model-00024-of-00026.safetensors": { + "sha256": "fd381409ec88968f177fc2968c1b311ec1bad48a8efaace6104918c439304758", + "size": 3370808752 + }, + "model-00025-of-00026.safetensors": { + "sha256": "4dc3961a2e10e6b01da2f19bf546b60e5fe8a520d219231b8ab38b618865fb13", + "size": 3832888256 + }, + "model-00026-of-00026.safetensors": { + "sha256": "a30d2c6c2a09f92bce458c4d42fbf4bc3410062d25f1a2d020ac42035c107b1a", + "size": 2231416848 + } + }, + "target_stats": { + "model.language_model.layers.0.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 30755, + "changed_fraction": 0.4692840576171875, + "elements": 65536, + "formula_nonzero_elements": 63488, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.0.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.25063905038321e-05, + "scaling": 2.0, + "shard": "model-00002-of-00026.safetensors" + }, + "model.language_model.layers.0.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 48709, + "changed_fraction": 0.7432403564453125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000579833984375, + "module": "base_model.model.model.layers.0.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.360447674059165e-05, + "scaling": 2.0, + "shard": "model-00002-of-00026.safetensors" + }, + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10751965, + "changed_fraction": 0.6408670544624329, + "elements": 16777216, + "formula_nonzero_elements": 16547840, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.0.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.51468740055135e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.0.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5165585, + "changed_fraction": 0.6157857179641724, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.0.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.136164236255221e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.0.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4227429, + "changed_fraction": 0.5039488077163696, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00115966796875, + "module": "base_model.model.model.layers.0.linear_attn.out_proj", + "rms_effective_bf16_delta": 5.702941032752764e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.1.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 33111, + "changed_fraction": 0.5052337646484375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000591278076171875, + "module": "base_model.model.model.layers.1.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 6.47281010775474e-05, + "scaling": 2.0, + "shard": "model-00002-of-00026.safetensors" + }, + "model.language_model.layers.1.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42255, + "changed_fraction": 0.6447601318359375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0003662109375, + "module": "base_model.model.model.layers.1.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.433693698526348e-05, + "scaling": 2.0, + "shard": "model-00002-of-00026.safetensors" + }, + "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 9998779, + "changed_fraction": 0.5959736704826355, + "elements": 16777216, + "formula_nonzero_elements": 16367615, + "max_abs_effective_bf16_delta": 0.0010216832160949707, + "module": "base_model.model.model.layers.1.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 6.986407024210961e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.1.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5231233, + "changed_fraction": 0.623611569404602, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.00091552734375, + "module": "base_model.model.model.layers.1.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.359320666631618e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.1.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4777187, + "changed_fraction": 0.569485068321228, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.1.linear_attn.out_proj", + "rms_effective_bf16_delta": 5.66848069458663e-05, + "scaling": 2.0, + "shard": "model-00001-of-00026.safetensors" + }, + "model.language_model.layers.10.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 41894, + "changed_fraction": 0.639251708984375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.10.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.030118138951974e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.10.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46368, + "changed_fraction": 0.70751953125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000560760498046875, + "module": "base_model.model.model.layers.10.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.305791432245633e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10747736, + "changed_fraction": 0.6406149864196777, + "elements": 16777216, + "formula_nonzero_elements": 16769023, + "max_abs_effective_bf16_delta": 0.0008058547973632812, + "module": "base_model.model.model.layers.10.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.884481753437252e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.10.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5717960, + "changed_fraction": 0.6816339492797852, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.10.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 8.197803025550057e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.10.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5380400, + "changed_fraction": 0.6413936614990234, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00091552734375, + "module": "base_model.model.model.layers.10.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.199138831694352e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.11.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 694448, + "changed_fraction": 0.6622772216796875, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.000579833984375, + "module": "base_model.model.model.layers.11.self_attn.k_proj", + "rms_effective_bf16_delta": 7.355739501388593e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.11.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5479007, + "changed_fraction": 0.6531485319137573, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.0009410381317138672, + "module": "base_model.model.model.layers.11.self_attn.o_proj", + "rms_effective_bf16_delta": 8.168481778209755e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.11.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10769572, + "changed_fraction": 0.6419165134429932, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.11.self_attn.q_proj", + "rms_effective_bf16_delta": 7.578644977082595e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.11.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 758420, + "changed_fraction": 0.7232856750488281, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.11.self_attn.v_proj", + "rms_effective_bf16_delta": 8.498003892171755e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.12.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42608, + "changed_fraction": 0.650146484375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.12.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.816997522647381e-05, + "scaling": 2.0, + "shard": "model-00009-of-00026.safetensors" + }, + "model.language_model.layers.12.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46557, + "changed_fraction": 0.7104034423828125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00054931640625, + "module": "base_model.model.model.layers.12.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.412433719402122e-05, + "scaling": 2.0, + "shard": "model-00009-of-00026.safetensors" + }, + "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10868056, + "changed_fraction": 0.6477866172790527, + "elements": 16777216, + "formula_nonzero_elements": 16777213, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.12.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.18584051920274e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.12.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5882139, + "changed_fraction": 0.7012056112289429, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.12.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 8.230692583350476e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.12.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5516697, + "changed_fraction": 0.6576415300369263, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.12.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.846577771796752e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.13.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 41837, + "changed_fraction": 0.6383819580078125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000946044921875, + "module": "base_model.model.model.layers.13.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 9.347651483814202e-05, + "scaling": 2.0, + "shard": "model-00009-of-00026.safetensors" + }, + "model.language_model.layers.13.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46031, + "changed_fraction": 0.7023773193359375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000396728515625, + "module": "base_model.model.model.layers.13.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.12724770719925e-05, + "scaling": 2.0, + "shard": "model-00009-of-00026.safetensors" + }, + "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10692640, + "changed_fraction": 0.6373310089111328, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.0012636184692382812, + "module": "base_model.model.model.layers.13.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.316534015371205e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.13.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5786985, + "changed_fraction": 0.6898623704910278, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0010528564453125, + "module": "base_model.model.model.layers.13.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 8.540713466959101e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.13.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5599044, + "changed_fraction": 0.6674580574035645, + "elements": 8388608, + "formula_nonzero_elements": 8388605, + "max_abs_effective_bf16_delta": 0.000885009765625, + "module": "base_model.model.model.layers.13.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.360459743314073e-05, + "scaling": 2.0, + "shard": "model-00008-of-00026.safetensors" + }, + "model.language_model.layers.14.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 36731, + "changed_fraction": 0.5604705810546875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.14.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 6.408820823201858e-05, + "scaling": 2.0, + "shard": "model-00011-of-00026.safetensors" + }, + "model.language_model.layers.14.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46324, + "changed_fraction": 0.70684814453125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000579833984375, + "module": "base_model.model.model.layers.14.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.931456048098573e-05, + "scaling": 2.0, + "shard": "model-00011-of-00026.safetensors" + }, + "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10758927, + "changed_fraction": 0.6412820219993591, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.00091552734375, + "module": "base_model.model.model.layers.14.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.396889941829679e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.14.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6000591, + "changed_fraction": 0.715326189994812, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009002685546875, + "module": "base_model.model.model.layers.14.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 8.636677447278575e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.14.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5598240, + "changed_fraction": 0.6673622131347656, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.14.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.746053163095078e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.15.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 701780, + "changed_fraction": 0.6692695617675781, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.0008392333984375, + "module": "base_model.model.model.layers.15.self_attn.k_proj", + "rms_effective_bf16_delta": 7.595614539234427e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.15.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5682420, + "changed_fraction": 0.6773972511291504, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.15.self_attn.o_proj", + "rms_effective_bf16_delta": 9.326062588130585e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.15.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11393837, + "changed_fraction": 0.6791256070137024, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.15.self_attn.q_proj", + "rms_effective_bf16_delta": 8.019703041610845e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.15.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 764574, + "changed_fraction": 0.7291545867919922, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.15.self_attn.v_proj", + "rms_effective_bf16_delta": 9.223723003081227e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.16.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 40458, + "changed_fraction": 0.617340087890625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.16.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.926234263603284e-05, + "scaling": 2.0, + "shard": "model-00011-of-00026.safetensors" + }, + "model.language_model.layers.16.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47306, + "changed_fraction": 0.721832275390625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.16.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.947450366004857e-05, + "scaling": 2.0, + "shard": "model-00011-of-00026.safetensors" + }, + "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10924269, + "changed_fraction": 0.6511371731758118, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.16.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.514577641788167e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.16.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5987956, + "changed_fraction": 0.7138199806213379, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.16.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.070390622614111e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.16.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5665606, + "changed_fraction": 0.6753928661346436, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.16.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.717200426859417e-05, + "scaling": 2.0, + "shard": "model-00010-of-00026.safetensors" + }, + "model.language_model.layers.17.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 40944, + "changed_fraction": 0.624755859375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.17.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.38633181213219e-05, + "scaling": 2.0, + "shard": "model-00013-of-00026.safetensors" + }, + "model.language_model.layers.17.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46734, + "changed_fraction": 0.713104248046875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.17.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.645919554666517e-05, + "scaling": 2.0, + "shard": "model-00013-of-00026.safetensors" + }, + "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11070108, + "changed_fraction": 0.65982985496521, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.17.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.773317292698886e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.17.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6084979, + "changed_fraction": 0.7253860235214233, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.17.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.123145477755034e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.17.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5673400, + "changed_fraction": 0.6763219833374023, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.17.linear_attn.out_proj", + "rms_effective_bf16_delta": 9.202713231300383e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.18.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42062, + "changed_fraction": 0.641815185546875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.18.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.428588345419565e-05, + "scaling": 2.0, + "shard": "model-00013-of-00026.safetensors" + }, + "model.language_model.layers.18.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47332, + "changed_fraction": 0.72222900390625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.18.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.459836919250263e-05, + "scaling": 2.0, + "shard": "model-00013-of-00026.safetensors" + }, + "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11072675, + "changed_fraction": 0.6599828600883484, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.18.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 9.043640688973794e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.18.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5985994, + "changed_fraction": 0.7135860919952393, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0013427734375, + "module": "base_model.model.model.layers.18.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.505438980045448e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.18.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5617121, + "changed_fraction": 0.6696130037307739, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000946044921875, + "module": "base_model.model.model.layers.18.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.487318640617153e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.19.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 717909, + "changed_fraction": 0.6846513748168945, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.19.self_attn.k_proj", + "rms_effective_bf16_delta": 7.58642914683779e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.19.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5519486, + "changed_fraction": 0.6579740047454834, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.19.self_attn.o_proj", + "rms_effective_bf16_delta": 8.263213799470523e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.19.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11628234, + "changed_fraction": 0.6930967569351196, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.19.self_attn.q_proj", + "rms_effective_bf16_delta": 8.709391886705632e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.19.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 761626, + "changed_fraction": 0.7263431549072266, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.0009212493896484375, + "module": "base_model.model.model.layers.19.self_attn.v_proj", + "rms_effective_bf16_delta": 8.91430244772417e-05, + "scaling": 2.0, + "shard": "model-00012-of-00026.safetensors" + }, + "model.language_model.layers.2.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 39811, + "changed_fraction": 0.6074676513671875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.2.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.184019530288684e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.2.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 44642, + "changed_fraction": 0.681182861328125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000579833984375, + "module": "base_model.model.model.layers.2.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.96682201680039e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10180888, + "changed_fraction": 0.6068282127380371, + "elements": 16777216, + "formula_nonzero_elements": 16547836, + "max_abs_effective_bf16_delta": 0.000995635986328125, + "module": "base_model.model.model.layers.2.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.100468354687935e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.2.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5378050, + "changed_fraction": 0.6411135196685791, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.2.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.283989786059287e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.2.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4510282, + "changed_fraction": 0.5376675128936768, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00093841552734375, + "module": "base_model.model.model.layers.2.linear_attn.out_proj", + "rms_effective_bf16_delta": 6.235573763046415e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.20.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 41909, + "changed_fraction": 0.6394805908203125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.20.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.93851098858551e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.20.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46827, + "changed_fraction": 0.7145233154296875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.20.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.938891775580925e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10428442, + "changed_fraction": 0.6215835809707642, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.20.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.930851044544183e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.20.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5788078, + "changed_fraction": 0.6899926662445068, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00115203857421875, + "module": "base_model.model.model.layers.20.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.368472263909851e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.20.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5360389, + "changed_fraction": 0.6390081644058228, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.20.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.357498774166259e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.21.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 40619, + "changed_fraction": 0.6197967529296875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.21.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.828153390238863e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.21.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 48063, + "changed_fraction": 0.7333831787109375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.21.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 8.272542708067484e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10417391, + "changed_fraction": 0.6209248900413513, + "elements": 16777216, + "formula_nonzero_elements": 16773120, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.21.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.82992226033504e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.21.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5771549, + "changed_fraction": 0.688022255897522, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.001621246337890625, + "module": "base_model.model.model.layers.21.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.694958700437348e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.21.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5266131, + "changed_fraction": 0.6277717351913452, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0007476806640625, + "module": "base_model.model.model.layers.21.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.394243018921675e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.22.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42877, + "changed_fraction": 0.6542510986328125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00054931640625, + "module": "base_model.model.model.layers.22.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.447058045108835e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.22.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45907, + "changed_fraction": 0.7004852294921875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000499725341796875, + "module": "base_model.model.model.layers.22.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.73146372057541e-05, + "scaling": 2.0, + "shard": "model-00015-of-00026.safetensors" + }, + "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10554381, + "changed_fraction": 0.6290901303291321, + "elements": 16777216, + "formula_nonzero_elements": 16769024, + "max_abs_effective_bf16_delta": 0.0010728836059570312, + "module": "base_model.model.model.layers.22.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.774579516589965e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.22.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5863367, + "changed_fraction": 0.6989678144454956, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00103759765625, + "module": "base_model.model.model.layers.22.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.434000093003106e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.22.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5293620, + "changed_fraction": 0.6310486793518066, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.22.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.249776368945531e-05, + "scaling": 2.0, + "shard": "model-00014-of-00026.safetensors" + }, + "model.language_model.layers.23.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 678159, + "changed_fraction": 0.6467428207397461, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.23.self_attn.k_proj", + "rms_effective_bf16_delta": 7.518396677676303e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.23.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5218370, + "changed_fraction": 0.6220781803131104, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.23.self_attn.o_proj", + "rms_effective_bf16_delta": 8.036047474386736e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.23.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10712835, + "changed_fraction": 0.6385347247123718, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.23.self_attn.q_proj", + "rms_effective_bf16_delta": 8.438226304057198e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.23.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 746834, + "changed_fraction": 0.7122364044189453, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.0007476806640625, + "module": "base_model.model.model.layers.23.self_attn.v_proj", + "rms_effective_bf16_delta": 8.178560910881905e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.24.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 40637, + "changed_fraction": 0.6200714111328125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.24.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 6.87900255946991e-05, + "scaling": 2.0, + "shard": "model-00017-of-00026.safetensors" + }, + "model.language_model.layers.24.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45379, + "changed_fraction": 0.6924285888671875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.24.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.982129294249513e-05, + "scaling": 2.0, + "shard": "model-00017-of-00026.safetensors" + }, + "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10362982, + "changed_fraction": 0.6176818609237671, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.24.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.51102613002069e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.24.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5883216, + "changed_fraction": 0.7013339996337891, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.24.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.154575677870438e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.24.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5249409, + "changed_fraction": 0.625778317451477, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.24.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.722079186994334e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.25.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 43391, + "changed_fraction": 0.6620941162109375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0007476806640625, + "module": "base_model.model.model.layers.25.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 9.756493403597129e-05, + "scaling": 2.0, + "shard": "model-00017-of-00026.safetensors" + }, + "model.language_model.layers.25.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47230, + "changed_fraction": 0.720672607421875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.25.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.860814471744432e-05, + "scaling": 2.0, + "shard": "model-00017-of-00026.safetensors" + }, + "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10332363, + "changed_fraction": 0.6158568263053894, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.001068115234375, + "module": "base_model.model.model.layers.25.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.739930282826853e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.25.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5786058, + "changed_fraction": 0.6897518634796143, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00103759765625, + "module": "base_model.model.model.layers.25.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.676194801627504e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.25.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5281483, + "changed_fraction": 0.6296018362045288, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00067138671875, + "module": "base_model.model.model.layers.25.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.813189765614977e-05, + "scaling": 2.0, + "shard": "model-00016-of-00026.safetensors" + }, + "model.language_model.layers.26.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 39782, + "changed_fraction": 0.607025146484375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.26.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.568452986791395e-05, + "scaling": 2.0, + "shard": "model-00019-of-00026.safetensors" + }, + "model.language_model.layers.26.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45840, + "changed_fraction": 0.699462890625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.26.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 8.063676498761255e-05, + "scaling": 2.0, + "shard": "model-00019-of-00026.safetensors" + }, + "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10485047, + "changed_fraction": 0.6249575018882751, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.26.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 8.989223925345269e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.26.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6003196, + "changed_fraction": 0.7156367301940918, + "elements": 8388608, + "formula_nonzero_elements": 8388604, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.26.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 9.987351500103812e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.26.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5295000, + "changed_fraction": 0.6312131881713867, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.26.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.994238414767838e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.27.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 683967, + "changed_fraction": 0.6522817611694336, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.0004901885986328125, + "module": "base_model.model.model.layers.27.self_attn.k_proj", + "rms_effective_bf16_delta": 7.73319782163451e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.27.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5308128, + "changed_fraction": 0.6327781677246094, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.27.self_attn.o_proj", + "rms_effective_bf16_delta": 8.711702037703368e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.27.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11430191, + "changed_fraction": 0.6812924742698669, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.001087188720703125, + "module": "base_model.model.model.layers.27.self_attn.q_proj", + "rms_effective_bf16_delta": 9.126735428806502e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.27.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 738739, + "changed_fraction": 0.7045164108276367, + "elements": 1048576, + "formula_nonzero_elements": 1048575, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.27.self_attn.v_proj", + "rms_effective_bf16_delta": 8.949030938293286e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.28.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 43108, + "changed_fraction": 0.65777587890625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00054931640625, + "module": "base_model.model.model.layers.28.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.5556712585574e-05, + "scaling": 2.0, + "shard": "model-00019-of-00026.safetensors" + }, + "model.language_model.layers.28.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47219, + "changed_fraction": 0.7205047607421875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.28.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.959844718254463e-05, + "scaling": 2.0, + "shard": "model-00019-of-00026.safetensors" + }, + "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10649242, + "changed_fraction": 0.6347442865371704, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.0014352798461914062, + "module": "base_model.model.model.layers.28.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 9.628502558009477e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.28.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5957038, + "changed_fraction": 0.7101342678070068, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.28.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010274432978587353, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.28.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5364574, + "changed_fraction": 0.6395070552825928, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.28.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.804179398761905e-05, + "scaling": 2.0, + "shard": "model-00018-of-00026.safetensors" + }, + "model.language_model.layers.29.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42884, + "changed_fraction": 0.65435791015625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.29.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.808322381583734e-05, + "scaling": 2.0, + "shard": "model-00021-of-00026.safetensors" + }, + "model.language_model.layers.29.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46574, + "changed_fraction": 0.710662841796875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.29.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 8.585155676340322e-05, + "scaling": 2.0, + "shard": "model-00021-of-00026.safetensors" + }, + "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10667129, + "changed_fraction": 0.6358104348182678, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.00128173828125, + "module": "base_model.model.model.layers.29.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 9.849695659035123e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.29.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6084440, + "changed_fraction": 0.7253217697143555, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00115966796875, + "module": "base_model.model.model.layers.29.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00011035977036764194, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.29.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5370665, + "changed_fraction": 0.6402331590652466, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.001007080078125, + "module": "base_model.model.model.layers.29.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.975065824565774e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.3.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 662135, + "changed_fraction": 0.6314611434936523, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.000682830810546875, + "module": "base_model.model.model.layers.3.self_attn.k_proj", + "rms_effective_bf16_delta": 7.241444047012491e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.3.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5041438, + "changed_fraction": 0.6009862422943115, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0005035400390625, + "module": "base_model.model.model.layers.3.self_attn.o_proj", + "rms_effective_bf16_delta": 6.026982563617216e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.3.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10113617, + "changed_fraction": 0.6028185486793518, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.00092315673828125, + "module": "base_model.model.model.layers.3.self_attn.q_proj", + "rms_effective_bf16_delta": 7.350570200486596e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.3.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 721849, + "changed_fraction": 0.6884088516235352, + "elements": 1048576, + "formula_nonzero_elements": 1048575, + "max_abs_effective_bf16_delta": 0.000926971435546875, + "module": "base_model.model.model.layers.3.self_attn.v_proj", + "rms_effective_bf16_delta": 7.747421220958643e-05, + "scaling": 2.0, + "shard": "model-00003-of-00026.safetensors" + }, + "model.language_model.layers.30.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 43839, + "changed_fraction": 0.6689300537109375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000885009765625, + "module": "base_model.model.model.layers.30.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 9.872395338817168e-05, + "scaling": 2.0, + "shard": "model-00021-of-00026.safetensors" + }, + "model.language_model.layers.30.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46826, + "changed_fraction": 0.714508056640625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000518798828125, + "module": "base_model.model.model.layers.30.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 8.24930908855796e-05, + "scaling": 2.0, + "shard": "model-00021-of-00026.safetensors" + }, + "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10720563, + "changed_fraction": 0.638995349407196, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.00112152099609375, + "module": "base_model.model.model.layers.30.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 9.983738297083213e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.30.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6054666, + "changed_fraction": 0.7217724323272705, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0014801025390625, + "module": "base_model.model.model.layers.30.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010544629155757812, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.30.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5355054, + "changed_fraction": 0.6383721828460693, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.30.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.681716448816516e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.31.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 696095, + "changed_fraction": 0.6638479232788086, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.0009212493896484375, + "module": "base_model.model.model.layers.31.self_attn.k_proj", + "rms_effective_bf16_delta": 8.437487298802524e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.31.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5254913, + "changed_fraction": 0.6264344453811646, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00128173828125, + "module": "base_model.model.model.layers.31.self_attn.o_proj", + "rms_effective_bf16_delta": 8.925954272837241e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.31.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11193374, + "changed_fraction": 0.6671770811080933, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.00146484375, + "module": "base_model.model.model.layers.31.self_attn.q_proj", + "rms_effective_bf16_delta": 9.08660571373039e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.31.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 744481, + "changed_fraction": 0.7099924087524414, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00115966796875, + "module": "base_model.model.model.layers.31.self_attn.v_proj", + "rms_effective_bf16_delta": 9.294866747722607e-05, + "scaling": 2.0, + "shard": "model-00020-of-00026.safetensors" + }, + "model.language_model.layers.32.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 42686, + "changed_fraction": 0.651336669921875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.32.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.927647995249258e-05, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.32.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 48385, + "changed_fraction": 0.7382965087890625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.000701904296875, + "module": "base_model.model.model.layers.32.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 9.253677487914579e-05, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.32.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10909815, + "changed_fraction": 0.6502756476402283, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0011749267578125, + "module": "base_model.model.model.layers.32.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 0.00010049900326370136, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.32.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5980567, + "changed_fraction": 0.7129391431808472, + "elements": 8388608, + "formula_nonzero_elements": 8388605, + "max_abs_effective_bf16_delta": 0.001491546630859375, + "module": "base_model.model.model.layers.32.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010056438913395777, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.32.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5391896, + "changed_fraction": 0.6427640914916992, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00093841552734375, + "module": "base_model.model.model.layers.32.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.832600221432643e-05, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.33.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 44844, + "changed_fraction": 0.68426513671875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.33.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 0.00010916238745063663, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.33.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 48673, + "changed_fraction": 0.7426910400390625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00055694580078125, + "module": "base_model.model.model.layers.33.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 9.246898753329683e-05, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.33.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10909430, + "changed_fraction": 0.6502526998519897, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.001220703125, + "module": "base_model.model.model.layers.33.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 0.00010028798313212791, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.33.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5999651, + "changed_fraction": 0.7152141332626343, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.001190185546875, + "module": "base_model.model.model.layers.33.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010424333896815218, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.33.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5184738, + "changed_fraction": 0.6180689334869385, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00115966796875, + "module": "base_model.model.model.layers.33.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.289390623967547e-05, + "scaling": 2.0, + "shard": "model-00022-of-00026.safetensors" + }, + "model.language_model.layers.34.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 40663, + "changed_fraction": 0.6204681396484375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0008544921875, + "module": "base_model.model.model.layers.34.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.94923818246955e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.34.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46075, + "changed_fraction": 0.7030487060546875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0003662109375, + "module": "base_model.model.model.layers.34.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.623493961533599e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.34.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10708613, + "changed_fraction": 0.6382830739021301, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.34.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 9.605744535491254e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.34.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5958854, + "changed_fraction": 0.710350751876831, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00115966796875, + "module": "base_model.model.model.layers.34.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010116970767495327, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.34.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5518636, + "changed_fraction": 0.6578726768493652, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.34.linear_attn.out_proj", + "rms_effective_bf16_delta": 9.38996420599654e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.35.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 690388, + "changed_fraction": 0.6584053039550781, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00054931640625, + "module": "base_model.model.model.layers.35.self_attn.k_proj", + "rms_effective_bf16_delta": 7.802179036060055e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.35.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5132304, + "changed_fraction": 0.6118183135986328, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.35.self_attn.o_proj", + "rms_effective_bf16_delta": 9.380541725705772e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.35.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11146707, + "changed_fraction": 0.6643955111503601, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0013427734375, + "module": "base_model.model.model.layers.35.self_attn.q_proj", + "rms_effective_bf16_delta": 8.689808335234204e-05, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.35.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 729376, + "changed_fraction": 0.695587158203125, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.001323699951171875, + "module": "base_model.model.model.layers.35.self_attn.v_proj", + "rms_effective_bf16_delta": 0.00010167408347723381, + "scaling": 2.0, + "shard": "model-00023-of-00026.safetensors" + }, + "model.language_model.layers.36.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 44847, + "changed_fraction": 0.6843109130859375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.36.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 0.00011075183923671724, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.36.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 50187, + "changed_fraction": 0.7657928466796875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.36.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 9.08482572719799e-05, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.36.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10660430, + "changed_fraction": 0.6354111433029175, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.001995086669921875, + "module": "base_model.model.model.layers.36.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 0.00010201097014785468, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.36.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6022287, + "changed_fraction": 0.7179125547409058, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.36.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010200691580254258, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.36.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5259156, + "changed_fraction": 0.6269402503967285, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.36.linear_attn.out_proj", + "rms_effective_bf16_delta": 8.514353110152134e-05, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.37.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 41306, + "changed_fraction": 0.630279541015625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.37.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 9.075526212392864e-05, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.37.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47667, + "changed_fraction": 0.7273406982421875, + "elements": 65536, + "formula_nonzero_elements": 65535, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.37.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 8.455174747374792e-05, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.37.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10697563, + "changed_fraction": 0.6376244425773621, + "elements": 16777216, + "formula_nonzero_elements": 16777213, + "max_abs_effective_bf16_delta": 0.0012664794921875, + "module": "base_model.model.model.layers.37.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 0.00010047640710325602, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.37.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6133336, + "changed_fraction": 0.7311506271362305, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.00140380859375, + "module": "base_model.model.model.layers.37.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00010564545033233658, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.37.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5651545, + "changed_fraction": 0.67371666431427, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0010833740234375, + "module": "base_model.model.model.layers.37.linear_attn.out_proj", + "rms_effective_bf16_delta": 0.00010830739000305004, + "scaling": 2.0, + "shard": "model-00024-of-00026.safetensors" + }, + "model.language_model.layers.38.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 41451, + "changed_fraction": 0.6324920654296875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.38.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 9.510995748775557e-05, + "scaling": 2.0, + "shard": "model-00025-of-00026.safetensors" + }, + "model.language_model.layers.38.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 46651, + "changed_fraction": 0.7118377685546875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00054931640625, + "module": "base_model.model.model.layers.38.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 9.034939394162375e-05, + "scaling": 2.0, + "shard": "model-00025-of-00026.safetensors" + }, + "model.language_model.layers.38.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10948974, + "changed_fraction": 0.6526097059249878, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.00146484375, + "module": "base_model.model.model.layers.38.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 0.00011139941384783658, + "scaling": 2.0, + "shard": "model-00025-of-00026.safetensors" + }, + "model.language_model.layers.38.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 6145820, + "changed_fraction": 0.7326388359069824, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00146484375, + "module": "base_model.model.model.layers.38.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 0.00011233124404451418, + "scaling": 2.0, + "shard": "model-00025-of-00026.safetensors" + }, + "model.language_model.layers.38.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5734733, + "changed_fraction": 0.6836334466934204, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.38.linear_attn.out_proj", + "rms_effective_bf16_delta": 0.00011524470527460758, + "scaling": 2.0, + "shard": "model-00025-of-00026.safetensors" + }, + "model.language_model.layers.39.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 744442, + "changed_fraction": 0.7099552154541016, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.39.self_attn.k_proj", + "rms_effective_bf16_delta": 7.563793790538429e-05, + "scaling": 2.0, + "shard": "model-00026-of-00026.safetensors" + }, + "model.language_model.layers.39.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5507689, + "changed_fraction": 0.6565676927566528, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.39.self_attn.o_proj", + "rms_effective_bf16_delta": 0.00010270824831516126, + "scaling": 2.0, + "shard": "model-00026-of-00026.safetensors" + }, + "model.language_model.layers.39.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 11262917, + "changed_fraction": 0.6713221669197083, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.0010986328125, + "module": "base_model.model.model.layers.39.self_attn.q_proj", + "rms_effective_bf16_delta": 8.799169598088403e-05, + "scaling": 2.0, + "shard": "model-00026-of-00026.safetensors" + }, + "model.language_model.layers.39.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 830406, + "changed_fraction": 0.7919368743896484, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.001190185546875, + "module": "base_model.model.model.layers.39.self_attn.v_proj", + "rms_effective_bf16_delta": 0.00011691026157727887, + "scaling": 2.0, + "shard": "model-00026-of-00026.safetensors" + }, + "model.language_model.layers.4.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 39980, + "changed_fraction": 0.61004638671875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.4.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 8.024908435996157e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.4.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45411, + "changed_fraction": 0.6929168701171875, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.4.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.820581114168836e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10472535, + "changed_fraction": 0.6242117285728455, + "elements": 16777216, + "formula_nonzero_elements": 16777216, + "max_abs_effective_bf16_delta": 0.000762939453125, + "module": "base_model.model.model.layers.4.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.423169572809558e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.4.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5458783, + "changed_fraction": 0.6507376432418823, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.000732421875, + "module": "base_model.model.model.layers.4.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.415020020046006e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.4.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4842240, + "changed_fraction": 0.577239990234375, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.4.linear_attn.out_proj", + "rms_effective_bf16_delta": 5.6341563341890654e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.5.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 36954, + "changed_fraction": 0.563873291015625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00067138671875, + "module": "base_model.model.model.layers.5.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.908453169709464e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.5.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45055, + "changed_fraction": 0.6874847412109375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00030517578125, + "module": "base_model.model.model.layers.5.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.656953782415742e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10373943, + "changed_fraction": 0.6183351874351501, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.5.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.285539636718919e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.5.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5398227, + "changed_fraction": 0.6435188055038452, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.5.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.219678309162771e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.5.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4970646, + "changed_fraction": 0.5925471782684326, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.5.linear_attn.out_proj", + "rms_effective_bf16_delta": 6.0648198292888544e-05, + "scaling": 2.0, + "shard": "model-00004-of-00026.safetensors" + }, + "model.language_model.layers.6.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 38780, + "changed_fraction": 0.59173583984375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0006103515625, + "module": "base_model.model.model.layers.6.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.117397091117397e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.6.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 43242, + "changed_fraction": 0.659820556640625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0003739595413208008, + "module": "base_model.model.model.layers.6.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.940429126071299e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10351489, + "changed_fraction": 0.6169968247413635, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.001117706298828125, + "module": "base_model.model.model.layers.6.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.364370105770223e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.6.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5560208, + "changed_fraction": 0.6628284454345703, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.6.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.529367979499003e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.6.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 4966000, + "changed_fraction": 0.5919933319091797, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.6.linear_attn.out_proj", + "rms_effective_bf16_delta": 5.961404818933803e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.7.self_attn.k_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 661506, + "changed_fraction": 0.6308612823486328, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.7.self_attn.k_proj", + "rms_effective_bf16_delta": 7.067536790705216e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.7.self_attn.o_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5113964, + "changed_fraction": 0.6096320152282715, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.7.self_attn.o_proj", + "rms_effective_bf16_delta": 7.080729971930253e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.7.self_attn.q_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10590889, + "changed_fraction": 0.631266176700592, + "elements": 16777216, + "formula_nonzero_elements": 16777215, + "max_abs_effective_bf16_delta": 0.0009765625, + "module": "base_model.model.model.layers.7.self_attn.q_proj", + "rms_effective_bf16_delta": 7.40017609793409e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.7.self_attn.v_proj.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 512, + 32 + ], + "base_shape": [ + 512, + 2048 + ], + "changed_elements": 719821, + "changed_fraction": 0.6864748001098633, + "elements": 1048576, + "formula_nonzero_elements": 1048576, + "max_abs_effective_bf16_delta": 0.000949859619140625, + "module": "base_model.model.model.layers.7.self_attn.v_proj", + "rms_effective_bf16_delta": 8.238527252719216e-05, + "scaling": 2.0, + "shard": "model-00005-of-00026.safetensors" + }, + "model.language_model.layers.8.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 36348, + "changed_fraction": 0.55462646484375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.0003662109375, + "module": "base_model.model.model.layers.8.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 6.411371916081861e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.8.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 47509, + "changed_fraction": 0.7249298095703125, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.8.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 6.999336439184671e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10457812, + "changed_fraction": 0.6233341693878174, + "elements": 16777216, + "formula_nonzero_elements": 16777214, + "max_abs_effective_bf16_delta": 0.0010771006345748901, + "module": "base_model.model.model.layers.8.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.903955210642053e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.8.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5633351, + "changed_fraction": 0.6715477705001831, + "elements": 8388608, + "formula_nonzero_elements": 8388607, + "max_abs_effective_bf16_delta": 0.00103759765625, + "module": "base_model.model.model.layers.8.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 8.130170606636331e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.8.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5311705, + "changed_fraction": 0.6332045793533325, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00079345703125, + "module": "base_model.model.model.layers.8.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.404814731691594e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.9.linear_attn.in_proj_a.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 39113, + "changed_fraction": 0.5968170166015625, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00048828125, + "module": "base_model.model.model.layers.9.linear_attn.in_proj_a", + "rms_effective_bf16_delta": 7.49421331488855e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.9.linear_attn.in_proj_b.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 32, + 32 + ], + "base_shape": [ + 32, + 2048 + ], + "changed_elements": 45887, + "changed_fraction": 0.7001800537109375, + "elements": 65536, + "formula_nonzero_elements": 65536, + "max_abs_effective_bf16_delta": 0.00042724609375, + "module": "base_model.model.model.layers.9.linear_attn.in_proj_b", + "rms_effective_bf16_delta": 7.018184983915064e-05, + "scaling": 2.0, + "shard": "model-00007-of-00026.safetensors" + }, + "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 8192, + 32 + ], + "base_shape": [ + 8192, + 2048 + ], + "changed_elements": 10470688, + "changed_fraction": 0.6241016387939453, + "elements": 16777216, + "formula_nonzero_elements": 16773118, + "max_abs_effective_bf16_delta": 0.000946044921875, + "module": "base_model.model.model.layers.9.linear_attn.in_proj_qkv", + "rms_effective_bf16_delta": 7.70431663340467e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.9.linear_attn.in_proj_z.weight": { + "a_shape": [ + 32, + 2048 + ], + "b_shape": [ + 4096, + 32 + ], + "base_shape": [ + 4096, + 2048 + ], + "changed_elements": 5617997, + "changed_fraction": 0.6697174310684204, + "elements": 8388608, + "formula_nonzero_elements": 8388608, + "max_abs_effective_bf16_delta": 0.00091552734375, + "module": "base_model.model.model.layers.9.linear_attn.in_proj_z", + "rms_effective_bf16_delta": 7.911620981435632e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + }, + "model.language_model.layers.9.linear_attn.out_proj.weight": { + "a_shape": [ + 32, + 4096 + ], + "b_shape": [ + 2048, + 32 + ], + "base_shape": [ + 2048, + 4096 + ], + "changed_elements": 5346855, + "changed_fraction": 0.6373947858810425, + "elements": 8388608, + "formula_nonzero_elements": 8388606, + "max_abs_effective_bf16_delta": 0.0007781982421875, + "module": "base_model.model.model.layers.9.linear_attn.out_proj", + "rms_effective_bf16_delta": 7.257326413407083e-05, + "scaling": 2.0, + "shard": "model-00006-of-00026.safetensors" + } + } +}